import numpy as np
import time
import sys

# 性能对比：Python 列表 vs NumPy 数组
size = 1_000_000

# Python 列表
python_list = list(range(size))
start = time.perf_counter()
python_result = [x * 2 + 1 for x in python_list]
python_time = time.perf_counter() - start
python_memory = sys.getsizeof(python_list) + sum(sys.getsizeof(x) for x in python_list[:100]) / 100 * size

# NumPy 数组
numpy_array = np.arange(size)
start = time.perf_counter()
numpy_result = numpy_array * 2 + 1
numpy_time = time.perf_counter() - start
numpy_memory = numpy_array.nbytes

print("=== Python 列表 vs NumPy 数组 ===")
print(f"Python 列表:")
print(f"  执行时间: {python_time:.4f} 秒")
print(f"  内存占用: {python_memory / 1024 / 1024:.2f} MB")
print(f"\nNumPy 数组:")
print(f"  执行时间: {numpy_time:.4f} 秒")
print(f"  内存占用: {numpy_memory / 1024 / 1024:.2f} MB")
print(f"\nNumPy 快 {python_time / numpy_time:.1f} 倍，节省内存 {(1 - numpy_memory / python_memory) * 100:.1f}%")

import numpy as np

# 1. 创建数组的多种方式
print("=== 创建 NumPy 数组 ===")

# 从列表创建
arr1 = np.array([1, 2, 3, 4, 5])
print(f"从列表创建: {arr1}")

# 创建特殊数组
print(f"全零数组: {np.zeros(5)}")
print(f"全一数组: {np.ones(5)}")
print(f"空数组: {np.empty(3)}")  # 未初始化，内容随机
print(f"单位矩阵:\n{np.eye(3)}")
print(f"对角线数组: {np.diag([1, 2, 3])}")

# 创建序列数组
print(f"arange (步长): {np.arange(0, 10, 2)}")
print(f"linspace (等间距): {np.linspace(0, 1, 5)}")
print(f"logspace (对数等间距): {np.logspace(0, 2, 5)}")

# 随机数组
np.random.seed(42)  # 设置随机种子，保证结果可重现
print(f"均匀分布: {np.random.rand(5)}")
print(f"标准正态分布: {np.random.randn(5)}")
print(f"随机整数: {np.random.randint(1, 10, 5)}")

# 2. 数组属性
arr = np.array([[1, 2, 3], [4, 5, 6]])
print(f"\n=== 数组属性 ===")
print(f"形状: {arr.shape}")
print(f"维度: {arr.ndim}")
print(f"元素个数: {arr.size}")
print(f"数据类型: {arr.dtype}")
print(f"每个元素字节数: {arr.itemsize}")
print(f"总字节数: {arr.nbytes}")

# 3. 数据类型
print(f"\n=== 数据类型 ===")
arr_float = np.array([1, 2, 3], dtype=np.float32)
arr_int = np.array([1.5, 2.5, 3.5], dtype=np.int32)  # 截断
arr_complex = np.array([1+2j, 3+4j])
arr_bool = np.array([True, False, True])

print(f"float32: {arr_float} ({arr_float.dtype})")
print(f"int32 (截断): {arr_int} ({arr_int.dtype})")
print(f"复数: {arr_complex} ({arr_complex.dtype})")
print(f"布尔: {arr_bool} ({arr_bool.dtype})")

# 类型转换
arr = np.array([1, 2, 3])
print(f"转换前: {arr.dtype}")
print(f"转换为 float: {arr.astype(np.float64).dtype}")

import numpy as np

print("=== 多维数组操作 ===\n")

# 1. 创建多维数组
matrix = np.array([[1, 2, 3],
                   [4, 5, 6],
                   [7, 8, 9]])
print(f"3x3 矩阵:\n{matrix}\n")

# 2. 重塑数组
arr = np.arange(12)
print(f"原始数组 (1D): {arr}")
print(f"重塑为 3x4:\n{arr.reshape(3, 4)}")
print(f"重塑为 2x2x3:\n{arr.reshape(2, 2, 3)}")
print(f"自动计算维度 (-1): {arr.reshape(3, -1).shape}")

# 3. 索引和切片
print(f"\n=== 索引和切片 ===")
arr = np.array([[1, 2, 3, 4],
                [5, 6, 7, 8],
                [9, 10, 11, 12]])

print(f"原始数组:\n{arr}")
print(f"第2行第3列: {arr[1, 2]}")
print(f"第1行: {arr[0]}")
print(f"第2列: {arr[:, 1]}")
print(f"前2行后2列:\n{arr[:2, 2:]}")
print(f"每隔一列:\n{arr[:, ::2]}")

# 4. 花式索引
print(f"\n=== 花式索引 ===")
arr = np.arange(10) * 2
print(f"数组: {arr}")
indices = [1, 3, 5, 7]
print(f"索引 {indices}: {arr[indices]}")

# 布尔索引
mask = arr > 10
print(f"大于10的元素: {arr[mask]}")

# 多维花式索引
matrix = np.arange(1, 10).reshape(3, 3)
print(f"矩阵:\n{matrix}")
rows = [0, 2]
cols = [1, 2]
print(f"指定行列: {matrix[rows][:, cols]}")

# 5. 数组操作
print(f"\n=== 数组操作 ===")
a = np.array([[1, 2], [3, 4]])
b = np.array([[5, 6], [7, 8]])

print(f"垂直堆叠:\n{np.vstack([a, b])}")
print(f"水平堆叠:\n{np.hstack([a, b])}")
print(f"深度堆叠:\n{np.dstack([a, b])}")

# 分割数组
arr = np.arange(10)
print(f"\n分割数组 {arr}:")
print(f"  split(3): {np.split(arr, 3)}")
print(f"  split([3, 7]): {np.split(arr, [3, 7])}")

# 转置
matrix = np.arange(6).reshape(2, 3)
print(f"\n转置:\n{matrix}")
print(f"转置后:\n{matrix.T}")

import numpy as np

print("=== 向量化运算 ===\n")

# 1. 基本算术运算
a = np.array([1, 2, 3, 4])
b = np.array([5, 6, 7, 8])

print(f"a = {a}")
print(f"b = {b}")
print(f"a + b = {a + b}")
print(f"a - b = {a - b}")
print(f"a * b = {a * b}")  # 逐元素乘法
print(f"a / b = {a / b}")
print(f"a ** 2 = {a ** 2}")
print(f"a % 2 = {a % 2}")

# 广播机制
print(f"\n=== 广播机制 ===")
a = np.array([[1, 2, 3], [4, 5, 6]])
b = np.array([10, 20, 30])
print(f"矩阵 a:\n{a}")
print(f"向量 b: {b}")
print(f"a + b:\n{a + b}")  # b 自动扩展为与 a 相同的形状

c = np.array([[10], [20]])
print(f"列向量 c:\n{c}")
print(f"a + c:\n{a + c}")

# 2. 通用函数 (ufunc)
print(f"\n=== 通用函数 ===")
arr = np.array([0, np.pi/2, np.pi, 3*np.pi/2])
print(f"角度: {arr}")
print(f"sin: {np.sin(arr)}")
print(f"cos: {np.cos(arr)}")
print(f"exp: {np.exp(arr)}")
print(f"log: {np.log(arr[1:])}")  # 注意：log(0) 是 -inf
print(f"sqrt: {np.sqrt(np.arange(1, 6))}")

# 3. 聚合函数
print(f"\n=== 聚合函数 ===")
arr = np.array([[1, 2, 3], [4, 5, 6]])
print(f"数组:\n{arr}")
print(f"总和: {arr.sum()} (列和: {arr.sum(axis=0)}, 行和: {arr.sum(axis=1)})")
print(f"平均值: {arr.mean():.2f}")
print(f"标准差: {arr.std():.2f}")
print(f"最小值: {arr.min()} (位置: {arr.argmin()})")
print(f"最大值: {arr.max()} (位置: {arr.argmax()})")
print(f"累积和: {arr.cumsum()}")
print(f"累积积: {arr.cumprod()}")

# 4. 条件逻辑
print(f"\n=== 条件逻辑 ===")
arr = np.array([1, 2, 3, 4, 5])
print(f"数组: {arr}")
print(f"arr > 3: {arr > 3}")
print(f"arr[arr > 3]: {arr[arr > 3]}")
print(f"np.where(arr > 3, arr * 2, arr): {np.where(arr > 3, arr * 2, arr)}")

# 复杂条件
conditions = [arr < 3, (arr >= 3) & (arr < 5), arr >= 5]
choices = [arr * 10, arr * 100, arr * 1000]
print(f"np.select: {np.select(conditions, choices)}")

# 5. 集合运算
print(f"\n=== 集合运算 ===")
a = np.array([1, 2, 3, 4, 5])
b = np.array([3, 4, 5, 6, 7])
print(f"a: {a}")
print(f"b: {b}")
print(f"交集: {np.intersect1d(a, b)}")
print(f"并集: {np.union1d(a, b)}")
print(f"差集 (a-b): {np.setdiff1d(a, b)}")
print(f"异或: {np.setxor1d(a, b)}")
print(f"b 的元素是否在 a 中: {np.in1d(b, a)}")

import pandas as pd
import numpy as np

print("=== Pandas Series ===\n")

# 1. 创建 Series
# 从列表创建
s1 = pd.Series([1, 2, 3, 4, 5])
print(f"从列表创建 (默认索引):\n{s1}\n")

# 指定索引
s2 = pd.Series([1, 2, 3, 4, 5], index=['a', 'b', 'c', 'd', 'e'])
print(f"指定索引:\n{s2}\n")

# 从字典创建
s3 = pd.Series({'a': 1, 'b': 2, 'c': 3})
print(f"从字典创建:\n{s3}\n")

# 从标量创建
s4 = pd.Series(5, index=['a', 'b', 'c'])
print(f"从标量创建:\n{s4}\n")

# 2. Series 属性
print("=== Series 属性 ===")
print(f"值: {s2.values}")
print(f"索引: {s2.index}")
print(f"数据类型: {s2.dtype}")
print(f"形状: {s2.shape}")
print(f"元素个数: {s2.size}")
print(f"是否有空值: {s2.hasnans}")

# 3. Series 索引和切片
print(f"\n=== Series 索引 ===")
print(f"标签索引 'b': {s2['b']}")
print(f"位置索引 2: {s2.iloc[2]}")
print(f"切片 (标签):\n{s2['b':'d']}")
print(f"切片 (位置):\n{s2.iloc[1:4]}")

# 布尔索引
print(f"值大于3的:\n{s2[s2 > 3]}")

# 4. Series 运算
print(f"\n=== Series 运算 ===")
s_a = pd.Series([1, 2, 3], index=['a', 'b', 'c'])
s_b = pd.Series([4, 5, 6], index=['b', 'c', 'd'])
print(f"s_a:\n{s_a}")
print(f"s_b:\n{s_b}")
print(f"s_a + s_b:\n{s_a + s_b}")  # 自动对齐索引，缺失值为 NaN

# 填充缺失值
print(f"加法 (填充0):\n{s_a.add(s_b, fill_value=0)}")

import pandas as pd
import numpy as np

print("=== Pandas DataFrame ===\n")

# 1. 创建 DataFrame
# 从字典创建
data = {
    'name': ['Alice', 'Bob', 'Charlie', 'David'],
    'age': [25, 30, 35, 28],
    'city': ['NYC', 'LA', 'Chicago', 'Boston'],
    'salary': [50000, 60000, 75000, 55000]
}
df = pd.DataFrame(data)
print(f"从字典创建:\n{df}\n")

# 从列表创建
data_list = [
    ['Alice', 25, 'NYC', 50000],
    ['Bob', 30, 'LA', 60000],
    ['Charlie', 35, 'Chicago', 75000],
    ['David', 28, 'Boston', 55000]
]
df2 = pd.DataFrame(data_list, columns=['name', 'age', 'city', 'salary'])
print(f"从列表创建:\n{df2}\n")

# 从 NumPy 数组创建
array = np.random.randn(5, 3)
df3 = pd.DataFrame(array, columns=['A', 'B', 'C'])
print(f"从 NumPy 数组创建:\n{df3}\n")

# 2. DataFrame 基本信息
print("=== DataFrame 信息 ===")
print(f"形状: {df.shape}")
print(f"列名: {df.columns.tolist()}")
print(f"索引: {df.index.tolist()}")
print(f"数据类型:\n{df.dtypes}")
print(f"基本信息:")
df.info()

# 3. 查看数据
print(f"\n=== 查看数据 ===")
print(f"前2行:\n{df.head(2)}")
print(f"后2行:\n{df.tail(2)}")
print(f"随机抽样1行:\n{df.sample(1)}")
print(f"描述性统计:\n{df.describe()}")
print(f"非空值计数:\n{df.count()}")

import pandas as pd

# 创建示例数据
df = pd.DataFrame({
    'name': ['Alice', 'Bob', 'Charlie', 'David', 'Eve'],
    'age': [25, 30, 35, 28, 32],
    'city': ['NYC', 'LA', 'Chicago', 'Boston', 'NYC'],
    'salary': [50000, 60000, 75000, 55000, 65000],
    'department': ['IT', 'HR', 'IT', 'Finance', 'HR']
})
print(f"原始数据:\n{df}\n")

print("=== DataFrame 索引与选择 ===\n")

# 1. 选择列
print("选择单列 (返回 Series):")
print(df['name'])
print("\n选择多列 (返回 DataFrame):")
print(df[['name', 'age']])

# 2. 选择行 - loc (标签索引)
print("\n=== loc - 标签索引 ===")
print("单行 (索引0):")
print(df.loc[0])
print("\n多行和指定列:")
print(df.loc[1:3, ['name', 'salary']])
print("\n条件选择:")
print(df.loc[df['age'] > 30])

# 3. 选择行 - iloc (位置索引)
print("\n=== iloc - 位置索引 ===")
print("前3行:")
print(df.iloc[:3])
print("\n特定位置:")
print(df.iloc[1:4, 0:3])
print("\n最后一行:")
print(df.iloc[-1])

# 4. 条件选择 - 布尔索引
print("\n=== 布尔索引 ===")
print("年龄 > 30 且 城市是 NYC:")
print(df[(df['age'] > 30) & (df['city'] == 'NYC')])
print("\n部门是 IT 或 HR:")
print(df[df['department'].isin(['IT', 'HR'])])

# 5. 使用 query 方法 (更简洁)
print("\n=== query 方法 ===")
print("薪资 > 60000:")
print(df.query('salary > 60000'))
print("\n复杂条件:")
print(df.query('age > 30 and department == "IT"'))

# 6. at 和 iat (快速标量访问)
print("\n=== at / iat - 标量访问 ===")
print(f"df.at[2, 'name']: {df.at[2, 'name']}")
print(f"df.iat[1, 2]: {df.iat[1, 2]}")

# 7. 设置值
df_copy = df.copy()
df_copy.loc[0, 'salary'] = 55000
df_copy.iloc[1, 1] = 31
print(f"\n修改后:\n{df_copy}")

import pandas as pd
import numpy as np

print("=== 数据清洗与处理 ===\n")

# 创建包含缺失值的数据
df = pd.DataFrame({
    'name': ['Alice', 'Bob', 'Charlie', 'David', 'Eve'],
    'age': [25, np.nan, 35, 28, 32],
    'city': ['NYC', 'LA', np.nan, 'Boston', 'NYC'],
    'salary': [50000, 60000, 75000, np.nan, 65000],
    'score': [85.5, 90.0, 88.5, 92.0, np.nan]
})
print(f"原始数据 (含缺失值):\n{df}\n")

# 1. 处理缺失值
print("=== 处理缺失值 ===")

# 检测缺失值
print(f"缺失值统计:\n{df.isnull().sum()}")
print(f"缺失值占比:\n{df.isnull().mean() * 100:.2f}%")

# 删除缺失值
print(f"\n删除任何包含缺失值的行:\n{df.dropna()}")
print(f"\n删除指定列有空值的行:\n{df.dropna(subset=['age', 'salary'])}")
print(f"\n删除全部为空的列:\n{df.dropna(axis=1, how='all')}")

# 填充缺失值
print(f"\n填充固定值:\n{df.fillna(0)}")
print(f"\n前向填充:\n{df.fillna(method='ffill')}")
print(f"\n用平均值填充 age:\n{df.assign(age=df['age'].fillna(df['age'].mean()))}")
print(f"\n用中位数填充 salary:\n{df.assign(salary=df['salary'].fillna(df['salary'].median()))}")

# 插值
df_interpolated = df.copy()
df_interpolated['score'] = df_interpolated['score'].interpolate()
print(f"\n线性插值:\n{df_interpolated}")

# 2. 重复值处理
print("\n=== 重复值处理 ===")
df_dup = pd.DataFrame({
    'A': [1, 2, 2, 3, 3, 3],
    'B': ['a', 'b', 'b', 'c', 'c', 'c']
})
print(f"含重复值:\n{df_dup}")
print(f"检测重复行:\n{df_dup.duplicated()}")
print(f"删除重复行:\n{df_dup.drop_duplicates()}")
print(f"按 A 列去重 (保留第一个):\n{df_dup.drop_duplicates(subset=['A'], keep='first')}")

# 3. 数据类型转换
print("\n=== 数据类型转换 ===")
df_types = pd.DataFrame({
    'int_col': ['1', '2', '3'],
    'float_col': ['1.5', '2.5', '3.5'],
    'date_col': ['2024-01-01', '2024-01-02', '2024-01-03'],
    'category_col': ['A', 'B', 'A']
})
print(f"原始数据类型:\n{df_types.dtypes}")

df_types['int_col'] = df_types['int_col'].astype(int)
df_types['float_col'] = df_types['float_col'].astype(float)
df_types['date_col'] = pd.to_datetime(df_types['date_col'])
df_types['category_col'] = df_types['category_col'].astype('category')
print(f"\n转换后数据类型:\n{df_types.dtypes}")

# 使用 to_numeric (安全转换)
s = pd.Series(['1', '2', 'invalid', '3'])
print(f"\n安全转换 (错误变为 NaN):\n{pd.to_numeric(s, errors='coerce')}")

import pandas as pd
import numpy as np

print("=== 数据变换与特征工程 ===\n")

# 创建示例数据
df = pd.DataFrame({
    'name': ['Alice', 'Bob', 'Charlie', 'David'],
    'birth_date': ['1990-05-15', '1985-10-20', '1992-03-08', '1988-12-01'],
    'salary': [50000, 60000, 75000, 55000],
    'department': ['IT', 'HR', 'IT', 'Finance'],
    'performance': [4.2, 3.8, 4.5, 4.0]
})
print(f"原始数据:\n{df}\n")

# 1. 使用 apply 函数
print("=== apply 函数 ===")
# 对单列应用函数
df['salary_k'] = df['salary'].apply(lambda x: x / 1000)
print(f"薪资 (千):\n{df[['name', 'salary', 'salary_k']]}")

# 对多列应用函数
df['age'] = pd.to_datetime('today').year - pd.to_datetime(df['birth_date']).dt.year
print(f"\n计算年龄:\n{df[['name', 'birth_date', 'age']]}")

# 对整行应用函数
df['bonus'] = df.apply(lambda row: row['salary'] * 0.1 if row['performance'] > 4.0 else row['salary'] * 0.05, axis=1)
print(f"\n计算奖金:\n{df[['name', 'performance', 'salary', 'bonus']]}")

# 2. 使用 map 和 replace
print("\n=== map 和 replace ===")
# map - 用于 Series
dept_map = {'IT': 'Technology', 'HR': 'Human Resources', 'Finance': 'Financial'}
df['dept_full'] = df['department'].map(dept_map)
print(f"部门映射:\n{df[['department', 'dept_full']]}")

# replace - 替换值
df_grade = df['performance'].copy()
df_grade = df_grade.replace({4.5: 'A', 4.2: 'B+', 4.0: 'B', 3.8: 'B-'})
print(f"\n绩效等级:\n{df_grade}")

# 3. 使用 transform (与 groupby 配合)
print("\n=== transform ===")
df['dept_avg_salary'] = df.groupby('department')['salary'].transform('mean')
df['salary_diff'] = df['salary'] - df['dept_avg_salary']
print(f"部门平均薪资和差异:\n{df[['name', 'department', 'salary', 'dept_avg_salary', 'salary_diff']]}")

# 4. 离散化和分箱
print("\n=== 离散化 ===")
# cut - 等宽分箱
df['age_group'] = pd.cut(df['age'], bins=[0, 30, 35, 50], labels=['Young', 'Mid', 'Senior'])
print(f"年龄分组:\n{df[['name', 'age', 'age_group']]}")

# qcut - 等频分箱
df['salary_quartile'] = pd.qcut(df['salary'], q=4, labels=['Q1', 'Q2', 'Q3', 'Q4'])
print(f"\n薪资四分位:\n{df[['name', 'salary', 'salary_quartile']]}")

# 5. 独热编码
print("\n=== 独热编码 ===")
dummies = pd.get_dummies(df['department'], prefix='dept')
df_encoded = pd.concat([df, dummies], axis=1)
print(f"独热编码后:\n{df_encoded[['name', 'department', 'dept_IT', 'dept_HR', 'dept_Finance']]}")

# 6. 字符串操作
print("\n=== 字符串操作 ===")
df['email'] = df['name'].str.lower() + '@company.com'
print(f"生成邮箱:\n{df[['name', 'email']]}")

# 字符串分割和提取
text_series = pd.Series(['Alice-IT-NYC', 'Bob-HR-LA', 'Charlie-IT-Chicago'])
split_df = text_series.str.split('-', expand=True)
split_df.columns = ['name', 'dept', 'city']
print(f"\n字符串分割:\n{split_df}")

import pandas as pd
import numpy as np

print("=== 数据聚合与分组 ===\n")

# 创建示例数据
np.random.seed(42)
df = pd.DataFrame({
    'date': pd.date_range('2024-01-01', periods=100, freq='D'),
    'product': np.random.choice(['A', 'B', 'C'], 100),
    'category': np.random.choice(['Electronics', 'Clothing', 'Food'], 100),
    'sales': np.random.randint(100, 1000, 100),
    'quantity': np.random.randint(1, 10, 100),
    'region': np.random.choice(['North', 'South', 'East', 'West'], 100)
})
df['revenue'] = df['sales'] * df['quantity']
print(f"原始数据 (前10行):\n{df.head(10)}\n")

# 1. groupby 基础
print("=== groupby 基础 ===")

# 单列分组
grouped = df.groupby('product')
print(f"按产品分组:")
print(f"  分组数量: {grouped.ngroups}")
print(f"  各组大小:\n{grouped.size()}")
print(f"  销售额统计:\n{grouped['revenue'].agg(['mean', 'sum', 'count'])}")

# 多列分组
grouped_multi = df.groupby(['category', 'region'])
print(f"\n按类别和地区分组:\n{grouped_multi['revenue'].mean().unstack()}")

# 2. 聚合函数
print("\n=== 聚合函数 ===")

# 单个聚合函数
print(f"产品平均销售额:\n{df.groupby('product')['revenue'].mean()}")

# 多个聚合函数
agg_result = df.groupby('product').agg({
    'revenue': ['sum', 'mean', 'std'],
    'quantity': ['sum', 'mean'],
    'sales': ['min', 'max']
})
print(f"\n多列多聚合:\n{agg_result}")

# 自定义聚合函数
def range_func(x):
    return x.max() - x.min()

custom_agg = df.groupby('category').agg({
    'revenue': ['sum', 'mean', range_func]
})
print(f"\n自定义聚合 (range):\n{custom_agg}")

# 3. 变换和过滤
print("\n=== 变换和过滤 ===")

# transform - 保持原形状
df['category_avg'] = df.groupby('category')['revenue'].transform('mean')
df['above_avg'] = df['revenue'] > df['category_avg']
print(f"添加类别平均值:\n{df[['category', 'revenue', 'category_avg', 'above_avg']].head(10)}")

# filter - 过滤组
high_performing_products = df.groupby('product').filter(lambda x: x['revenue'].mean() > 3000)
print(f"\n平均收入 > 3000 的产品:\n{high_performing_products['product'].unique()}")

# 4. 高级分组操作
print("\n=== 高级分组操作 ===")

# rolling - 滑动窗口
df_sorted = df.sort_values('date')
df_sorted['revenue_ma7'] = df_sorted.groupby('product')['revenue'].transform(
    lambda x: x.rolling(window=7, min_periods=1).mean()
)
print(f"7天滑动平均:\n{df_sorted[['date', 'product', 'revenue', 'revenue_ma7']].head(10)}")

# expanding - 扩展窗口
df_sorted['revenue_cumavg'] = df_sorted.groupby('product')['revenue'].transform(
    lambda x: x.expanding().mean()
)
print(f"\n累积平均:\n{df_sorted[['product', 'revenue', 'revenue_cumavg']].head(10)}")

# 5. 交叉表和数据透视表
print("\n=== 交叉表和透视表 ===")

# 交叉表 (频数统计)
crosstab = pd.crosstab(df['category'], df['region'], margins=True)
print(f"交叉表:\n{crosstab}")

# 透视表
pivot = pd.pivot_table(
    df,
    values='revenue',
    index='category',
    columns='region',
    aggfunc='mean',
    fill_value=0
)
print(f"\n透视表 (平均收入):\n{pivot}")

# 复杂透视表
pivot_complex = pd.pivot_table(
    df,
    values='revenue',
    index=['category', 'product'],
    columns='region',
    aggfunc={'revenue': ['mean', 'count']},
    fill_value=0
)
print(f"\n复杂透视表:\n{pivot_complex}")

import pandas as pd
import numpy as np
from datetime import datetime, timedelta

print("=== 时间序列处理 ===\n")

# 1. 创建时间序列
print("=== 创建时间序列 ===")

# date_range
dates = pd.date_range('2024-01-01', periods=10, freq='D')
print(f"日期范围 (天):\n{dates}")

dates_weekly = pd.date_range('2024-01-01', periods=5, freq='W')
print(f"\n日期范围 (周):\n{dates_weekly}")

dates_business = pd.date_range('2024-01-01', periods=5, freq='B')
print(f"\n工作日:\n{dates_business}")

dates_hourly = pd.date_range('2024-01-01', periods=5, freq='4H')
print(f"\n每4小时:\n{dates_hourly}")

# 创建带时间索引的 DataFrame
ts_df = pd.DataFrame({
    'value': np.random.randn(10),
    'category': np.random.choice(['A', 'B'], 10)
}, index=dates)
print(f"\n时间序列 DataFrame:\n{ts_df}")

# 2. 时间索引操作
print("\n=== 时间索引操作 ===")

# 创建更大的时间序列
dates_full = pd.date_range('2024-01-01', '2024-12-31', freq='D')
ts_data = pd.DataFrame({
    'sales': np.random.randint(100, 500, len(dates_full)),
    'customers': np.random.randint(10, 50, len(dates_full))
}, index=dates_full)

print(f"全年数据形状: {ts_data.shape}")

# 基于时间的索引
print(f"\n2024年1月的数据:\n{ts_data['2024-01']}")
print(f"\n2024年第二季度数据:\n{ts_data['2024-04':'2024-06']}")
print(f"\n特定日期数据:\n{ts_data.loc['2024-07-15']}")

# 3. 重采样
print("\n=== 重采样 ===")

# 下采样 (高频到低频)
monthly = ts_data.resample('M').mean()
print(f"月度平均:\n{monthly.head()}")

quarterly_sum = ts_data.resample('Q').sum()
print(f"\n季度总和:\n{quarterly_sum}")

# 多种聚合
weekly_stats = ts_data.resample('W').agg({
    'sales': ['sum', 'mean', 'max'],
    'customers': 'sum'
})
print(f"\n周统计:\n{weekly_stats.head()}")

# 上采样 (低频到高频) - 需要插值
weekly_data = ts_data.resample('W').mean()
daily_upsampled = weekly_data.resample('D').interpolate(method='linear')
print(f"\n上采样 (周到天，线性插值):\n{daily_upsampled.head(10)}")

# 4. 时间偏移和移动
print("\n=== 时间偏移和移动 ===")

# shift - 移动数据
ts_data['sales_lag1'] = ts_data['sales'].shift(1)
ts_data['sales_lag7'] = ts_data['sales'].shift(7)
print(f"滞后值:\n{ts_data[['sales', 'sales_lag1', 'sales_lag7']].head(10)}")

# diff - 差分
ts_data['sales_diff'] = ts_data['sales'].diff()
ts_data['sales_pct_change'] = ts_data['sales'].pct_change()
print(f"\n差分和百分比变化:\n{ts_data[['sales', 'sales_diff', 'sales_pct_change']].head(10)}")

# 5. 时间特征提取
print("\n=== 时间特征提取 ===")

ts_data['year'] = ts_data.index.year
ts_data['month'] = ts_data.index.month
ts_data['day'] = ts_data.index.day
ts_data['dayofweek'] = ts_data.index.dayofweek
ts_data['quarter'] = ts_data.index.quarter
ts_data['week_of_year'] = ts_data.index.isocalendar().week
ts_data['is_weekend'] = ts_data.index.dayofweek >= 5

print(f"时间特征:\n{ts_data[['year', 'month', 'day', 'dayofweek', 'quarter', 'week_of_year', 'is_weekend']].head(10)}")

# 6. 时间序列分析示例
print("\n=== 时间序列分析示例 ===")

# 计算移动平均
ts_data['MA7'] = ts_data['sales'].rolling(window=7).mean()
ts_data['MA30'] = ts_data['sales'].rolling(window=30).mean()

# 计算累积统计
ts_data['cumsum'] = ts_data['sales'].cumsum()
ts_data['cummax'] = ts_data['sales'].cummax()

print(f"移动平均和累积统计:\n{ts_data[['sales', 'MA7', 'MA30', 'cumsum', 'cummax']].head(15)}")

# 检测异常值 (基于 Z-score)
ts_data['sales_zscore'] = (ts_data['sales'] - ts_data['sales'].mean()) / ts_data['sales'].std()
ts_data['is_outlier'] = abs(ts_data['sales_zscore']) > 2
outliers = ts_data[ts_data['is_outlier']]
print(f"\n异常值检测 (|Z-score| > 2):\n{len(outliers)} 个异常值")

import pandas as pd
import numpy as np
import time

print("=== 性能优化技巧 ===\n")

# 创建大数据集
n = 1_000_000
df = pd.DataFrame({
    'A': np.random.randn(n),
    'B': np.random.randn(n),
    'C': np.random.choice(['X', 'Y', 'Z'], n),
    'D': np.random.randint(1, 100, n)
})

# 1. 向量化 vs 循环
print("=== 向量化 vs 循环 ===")

# 循环方式 (慢)
start = time.perf_counter()
result_loop = []
for i in range(len(df)):
    if df.iloc[i]['A'] > 0:
        result_loop.append(df.iloc[i]['A'] * df.iloc[i]['B'])
    else:
        result_loop.append(0)
loop_time = time.perf_counter() - start

# 向量化方式 (快)
start = time.perf_counter()
result_vectorized = np.where(df['A'] > 0, df['A'] * df['B'], 0)
vectorized_time = time.perf_counter() - start

print(f"循环方式: {loop_time:.4f} 秒")
print(f"向量化方式: {vectorized_time:.4f} 秒")
print(f"向量化快 {loop_time / vectorized_time:.1f} 倍")

# 2. 使用 categorical 类型节省内存
print("\n=== 内存优化 ===")
df_test = df.copy()
memory_before = df_test['C'].memory_usage(deep=True)
df_test['C'] = df_test['C'].astype('category')
memory_after = df_test['C'].memory_usage(deep=True)

print(f"转换前内存: {memory_before:,} 字节")
print(f"转换后内存: {memory_after:,} 字节")
print(f"节省: {(1 - memory_after / memory_before) * 100:.1f}%")

# 3. 使用 eval 和 query
print("\n=== eval 和 query ===")

# 传统方式
start = time.perf_counter()
df['E'] = df['A'] + df['B'] * 2
df_filtered = df[(df['A'] > 0) & (df['D'] > 50)]
traditional_time = time.perf_counter() - start

# 使用 eval 和 query
start = time.perf_counter()
df.eval('F = A + B * 2', inplace=True)
df_filtered_eval = df.query('A > 0 and D > 50')
eval_time = time.perf_counter() - start

print(f"传统方式: {traditional_time:.4f} 秒")
print(f"eval/query 方式: {eval_time:.4f} 秒")
print(f"eval/query 快 {traditional_time / eval_time:.1f} 倍")

# 4. 分块处理大文件
print("\n=== 分块处理示例 ===")
# 创建示例大文件
df.to_csv('large_file.csv', index=False)

chunk_size = 100000
chunk_results = []

for chunk in pd.read_csv('large_file.csv', chunksize=chunk_size):
    # 对每个块进行处理
    chunk_result = chunk[chunk['A'] > 0]['B'].mean()
    chunk_results.append(chunk_result)

overall_mean = np.mean(chunk_results)
print(f"分块处理完成，结果: {overall_mean:.4f}")

import pandas as pd
import numpy as np

print("=== 数据合并与连接 ===\n")

# 创建示例数据
customers = pd.DataFrame({
    'customer_id': [1, 2, 3, 4],
    'name': ['Alice', 'Bob', 'Charlie', 'David'],
    'city': ['NYC', 'LA', 'Chicago', 'Boston']
})

orders = pd.DataFrame({
    'order_id': [101, 102, 103, 104, 105],
    'customer_id': [1, 2, 1, 3, 5],
    'product': ['A', 'B', 'C', 'A', 'D'],
    'amount': [100, 200, 150, 300, 250]
})

print(f"客户表:\n{customers}\n")
print(f"订单表:\n{orders}\n")

# 1. merge - 类似 SQL JOIN
print("=== merge (JOIN) ===")

# 内连接
inner_join = pd.merge(customers, orders, on='customer_id', how='inner')
print(f"内连接 (只保留两边都有的):\n{inner_join}\n")

# 左连接
left_join = pd.merge(customers, orders, on='customer_id', how='left')
print(f"左连接 (保留所有客户):\n{left_join}\n")

# 右连接
right_join = pd.merge(customers, orders, on='customer_id', how='right')
print(f"右连接 (保留所有订单):\n{right_join}\n")

# 外连接
outer_join = pd.merge(customers, orders, on='customer_id', how='outer')
print(f"外连接 (保留所有记录):\n{outer_join}\n")

# 不同列名的连接
customers2 = customers.rename(columns={'customer_id': 'id'})
merged_diff = pd.merge(customers2, orders, left_on='id', right_on='customer_id')
print(f"不同列名连接:\n{merged_diff}\n")

# 2. concat - 轴向连接
print("=== concat ===")

# 垂直连接 (追加行)
df1 = pd.DataFrame({'A': [1, 2], 'B': [3, 4]})
df2 = pd.DataFrame({'A': [5, 6], 'B': [7, 8]})
vertical_concat = pd.concat([df1, df2], ignore_index=True)
print(f"垂直连接:\n{vertical_concat}\n")

# 水平连接 (追加列)
horizontal_concat = pd.concat([df1, df2], axis=1)
print(f"水平连接:\n{horizontal_concat}\n")

# 部分重叠的列
df3 = pd.DataFrame({'A': [1, 2], 'C': [9, 10]})
concat_partial = pd.concat([df1, df3], sort=False)
print(f"部分重叠列连接:\n{concat_partial}\n")

# 3. join - 基于索引的连接
print("=== join ===")

left = pd.DataFrame({'A': [1, 2, 3]}, index=['a', 'b', 'c'])
right = pd.DataFrame({'B': [4, 5, 6]}, index=['b', 'c', 'd'])
joined = left.join(right, how='outer')
print(f"基于索引的连接:\n{joined}\n")

# 4. 实战：合并多个数据源
print("=== 实战：合并多个数据源 ===")

# 模拟多个数据源
sales_2023 = pd.DataFrame({
    'month': ['Jan', 'Feb', 'Mar'],
    'sales': [1000, 1200, 1100]
})

sales_2024 = pd.DataFrame({
    'month': ['Jan', 'Feb', 'Mar'],
    'sales': [1200, 1300, 1400]
})

targets = pd.DataFrame({
    'month': ['Jan', 'Feb', 'Mar', 'Apr'],
    'target': [1000, 1100, 1200, 1300]
})

# 合并销售数据
all_sales = pd.concat([
    sales_2023.assign(year=2023),
    sales_2024.assign(year=2024)
], ignore_index=True)
print(f"合并后的销售数据:\n{all_sales}\n")

# 与目标合并
sales_with_targets = pd.merge(all_sales, targets, on='month', how='left')
sales_with_targets['achieved'] = sales_with_targets['sales'] >= sales_with_targets['target']
print(f"销售与目标对比:\n{sales_with_targets}")

import pandas as pd
import numpy as np
from datetime import datetime, timedelta

print("=== 实战案例：销售数据分析 ===\n")

# 1. 生成模拟数据
np.random.seed(42)

n_customers = 100
n_orders = 1000

# 客户数据
customers = pd.DataFrame({
    'customer_id': range(1, n_customers + 1),
    'signup_date': [datetime(2023, 1, 1) + timedelta(days=np.random.randint(0, 365)) 
                    for _ in range(n_customers)],
    'region': np.random.choice(['North', 'South', 'East', 'West'], n_customers),
    'segment': np.random.choice(['Retail', 'Wholesale', 'Online'], n_customers, p=[0.5, 0.2, 0.3])
})

# 产品数据
products = pd.DataFrame({
    'product_id': range(1, 21),
    'category': np.random.choice(['Electronics', 'Clothing', 'Food', 'Books'], 20),
    'price': np.random.uniform(10, 500, 20).round(2),
    'cost': np.random.uniform(5, 300, 20).round(2)
})
products['margin'] = products['price'] - products['cost']
products['margin_pct'] = (products['margin'] / products['price'] * 100).round(2)

# 订单数据
orders = pd.DataFrame({
    'order_id': range(1, n_orders + 1),
    'customer_id': np.random.choice(customers['customer_id'], n_orders),
    'product_id': np.random.choice(products['product_id'], n_orders),
    'quantity': np.random.randint(1, 5, n_orders),
    'order_date': [datetime(2024, 1, 1) + timedelta(days=np.random.randint(0, 90)) 
                   for _ in range(n_orders)]
})

print("数据加载完成")
print(f"客户数: {len(customers)}")
print(f"产品数: {len(products)}")
print(f"订单数: {len(orders)}\n")

# 2. 数据整合
print("=== 数据整合 ===")

# 合并订单与产品信息
order_details = orders.merge(products, on='product_id')
order_details = order_details.merge(customers, on='customer_id')

# 计算订单金额
order_details['revenue'] = order_details['quantity'] * order_details['price']
order_details['profit'] = order_details['quantity'] * order_details['margin']

print(f"合并后的数据形状: {order_details.shape}")
print(f"前5行数据:\n{order_details.head()}\n")

# 3. 业务分析
print("=== 业务分析 ===\n")

# 总体指标
total_revenue = order_details['revenue'].sum()
total_profit = order_details['profit'].sum()
avg_order_value = order_details['revenue'].mean()
profit_margin = (total_profit / total_revenue * 100)

print("总体指标:")
print(f"  总收入: ${total_revenue:,.2f}")
print(f"  总利润: ${total_profit:,.2f}")
print(f"  平均订单价值: ${avg_order_value:.2f}")
print(f"  利润率: {profit_margin:.2f}%\n")

# 按类别分析
category_analysis = order_details.groupby('category').agg({
    'revenue': 'sum',
    'profit': 'sum',
    'order_id': 'count'
}).rename(columns={'order_id': 'order_count'})

category_analysis['profit_margin'] = (category_analysis['profit'] / category_analysis['revenue'] * 100)
category_analysis['avg_order_value'] = category_analysis['revenue'] / category_analysis['order_count']

print("按产品类别分析:")
print(category_analysis.round(2))
print()

# 按地区分析
region_analysis = order_details.groupby('region').agg({
    'revenue': 'sum',
    'profit': 'sum',
    'customer_id': 'nunique',
    'order_id': 'count'
}).rename(columns={'customer_id': 'unique_customers', 'order_id': 'order_count'})

region_analysis['revenue_per_customer'] = region_analysis['revenue'] / region_analysis['unique_customers']

print("按地区分析:")
print(region_analysis.round(2))
print()

# 客户分层 (RFM 分析)
print("=== 客户分层 (RFM) ===")

current_date = order_details['order_date'].max() + timedelta(days=1)

rfm = order_details.groupby('customer_id').agg({
    'order_date': lambda x: (current_date - x.max()).days,  # Recency
    'order_id': 'count',  # Frequency
    'revenue': 'sum'  # Monetary
}).rename(columns={
    'order_date': 'recency',
    'order_id': 'frequency',
    'revenue': 'monetary'
})

# 评分
rfm['R_score'] = pd.qcut(rfm['recency'], 4, labels=['4', '3', '2', '1'])  # 越近越好
rfm['F_score'] = pd.qcut(rfm['frequency'].rank(method='first'), 4, labels=['1', '2', '3', '4'])
rfm['M_score'] = pd.qcut(rfm['monetary'], 4, labels=['1', '2', '3', '4'])

rfm['RFM_score'] = rfm['R_score'].astype(str) + rfm['F_score'].astype(str) + rfm['M_score'].astype(str)

print(f"RFM 分析结果 (前10个客户):\n{rfm.head(10)}\n")

# 客户分层
def segment_customer(row):
    if row['R_score'] in ['4', '3'] and row['F_score'] in ['4', '3'] and row['M_score'] in ['4', '3']:
        return 'Champions'
    elif row['R_score'] in ['4', '3'] and row['F_score'] in ['2', '1']:
        return 'New Customers'
    elif row['R_score'] in ['2', '1'] and row['F_score'] in ['4', '3'] and row['M_score'] in ['4', '3']:
        return 'At Risk'
    elif row['R_score'] in ['1'] and row['F_score'] in ['1'] and row['M_score'] in ['1']:
        return 'Lost'
    else:
        return 'Others'

rfm['segment'] = rfm.apply(segment_customer, axis=1)

segment_summary = rfm.groupby('segment').agg({
    'recency': 'mean',
    'frequency': 'mean',
    'monetary': 'mean',
    'segment': 'count'
}).rename(columns={'segment': 'count'}).round(2)

print("客户分层汇总:")
print(segment_summary)
print()

# 4. 时间趋势分析
print("=== 时间趋势分析 ===")

# 按天汇总
daily_sales = order_details.groupby(order_details['order_date'].dt.date).agg({
    'revenue': 'sum',
    'order_id': 'count',
    'profit': 'sum'
}).rename(columns={'order_id': 'order_count'})

print(f"日销售趋势 (前10天):\n{daily_sales.head(10)}\n")

# 计算移动平均
daily_sales['revenue_ma7'] = daily_sales['revenue'].rolling(window=7, min_periods=1).mean()
daily_sales['orders_ma7'] = daily_sales['order_count'].rolling(window=7, min_periods=1).mean()

print(f"带移动平均的销售趋势:\n{daily_sales.tail(10)}\n")

# 5. 产品分析
print("=== 产品分析 ===")

# 畅销产品
top_products = order_details.groupby(['product_id', 'category']).agg({
    'revenue': 'sum',
    'quantity': 'sum',
    'profit': 'sum'
}).sort_values('revenue', ascending=False).head(5)

print("TOP 5 畅销产品:")
print(top_products.round(2))
print()

# 产品关联分析 (简单的关联规则)
print("产品购买组合分析:")
customer_products = order_details.groupby('customer_id')['category'].apply(lambda x: '|'.join(sorted(set(x))))
product_combinations = customer_products.value_counts().head(5)
print(f"常见的产品组合:\n{product_combinations}")