- 1. 基本概念
- 2. Series
- 2.1 Series 创建
- 2.2 Series的切片和索引
- 2.3 pandas读取外部文件
- 3. DataFrame
- 3.1 DataFrame的创建
- 3.2 描述信息
- 3.3 排序
- 3.4 获取行列数据
- 4. 数据的合并和分组聚合
- 4.1 字符串离散化
- 4.2 join() 合并数据
- 4.3 merge() 合并数据
- 4.4 分组 .groupby()
- 4.5 聚合 .count()
- 5. 索引和复合索引
- 5.1 简单索引
- 5.2 Series复合索引
- 5.3 DataFrame复合索引
- 6. pandas 中的时间序列
- 6.1 基本操作
- 6.2 DataFrame中使用时间序列
- 6.3 重采样
- 6.4 PeriodIndex时间段
- 常用数据类型
- Series 一维，带标签数据；
- DataFrame 二维，Series 容器；
import pandas as pd
import numpy as np
# 传入可迭代数据之列表[]，创建Series
p1 = pd.Series([1, 2, 34, 32, 56, 788, 67])
# 传入数据data，同时传入索引index，创建Series
p2 = pd.Series([1, 2, 34, 56, 34, 12, 78], index=list("abcdefg"))
# 传入字典{}创建Series
temp_dict = {"name": "arina", "age": 18, "tel": 12345}
p3 = pd.Series(temp_dict)
2.2 Series的切片和索引
- 切片
import pandas as pd import numpy as np import string t = pd.Series(np.arange(10), index=list(string.ascii_uppercase[:10])) print(t) t1 = t[0:5] print(t1) ''' A 0 B 1 C 2 D 3 E 4 F 5 G 6 H 7 I 8 J 9 A 0 B 1 C 2 D 3 E 4 '''
- 索引
print(t[1]) # 1 print(t["A"]) # 0 t2 = t[['A', 'D', 'E']] ''' A 0 D 3 E 4 ''' print(t[t > 4]) ''' F 5 G 6 H 7 I 8 J 9 '''
- .index 返回索引；
- .values 返回值；
import pandas as pd df = pd.read_csv(filepath_or_buffer)
3. DataFrame 3.1 DataFrame的创建
- 传入可迭代对象
import pandas as pd import numpy as np df = pd.DataFrame(np.arange(12).reshape(3, 4)) print(df) ''' 0 1 2 3 0 0 1 2 3 1 4 5 6 7 2 8 9 10 11 '''
- 传入可迭代对象、index、columns
import pandas as pd
import numpy as np
df = pd.DataFrame(np.arange(12).reshape(3, 4), index=list("abc"), columns=list("WXYZ"))
print(df)
'''
W X Y Z
a 0 1 2 3
b 4 5 6 7
c 8 9 10 11
'''
- 传入列表创建DataFrame
import pandas as pd
import numpy as np
list1 = {"name": ["xiaoming", "xiaowang"], "age": [15, 23], "tel": [10000, 10001]}
df1 = pd.DataFrame(list1)
print(df1)
'''
name age tel
0 xiaoming 15 10000
1 xiaowang 23 10001
'''
# 传入的列表中某个键值对不存在时，返回NAN
list2 = [{"name": "Tom", "age": 18, "tel": 10089}, {"age": 22, "tel": 10078}, {"name": "Lucy", "tel": 12789}]
df2 = pd.DataFrame(list2)
print(df2)
'''
name age tel
0 Tom 18.0 10089
1 NaN 22.0 10078
2 Lucy NaN 12789
'''
3.2 描述信息
import pandas as pd
import numpy as np
list2 = [{"name": "Tom", "age": 18, "tel": 10089}, {"age": 22, "tel": 10078}, {"name": "Lucy", "tel": 12789}]
df2 = pd.DataFrame(list2)
print("DataFrame对象的行数、列数:", df2.shape)
print("DataFrame对象的列数据类型:", df2.dtypes)
print("DataFrame对象的数据维度:", df2.ndim)
print("DataFrame对象的行索引:", df2.index)
print("DataFrame对象的列索引:", df2.columns)
print("DataFrame对象的对象值:", df2.values)
df.head(3) # 显示头部几行，默认5行 df.tail(3) # 显示末尾几行，默认5行 df.info() # 相关信息概览：行数、列数、列索引、列非空值个数、列类型、内存占用 df.describe() # 快速综合统计结果：计数、均值、标准差、最大值、四分位数（x%的中位数）、最小值3.3 排序
import pandas as pd
import numpy as np
list2 = [{"name": "Tom", "age": 18, "tel": 10089}, {"name": "Lulu", "age": 34, "tel": 10078},
{"name": "Lucy", "age": 23, "tel": 12789}]
df2 = pd.DataFrame(list2)
print(df2)
print("排序：")
df_sort = df2.sort_values(by="age", ascending=False) #ascending默认是True，升序
print(df_sort)
'''
name age tel
0 Tom 18 10089
1 Lulu 34 10078
2 Lucy 23 12789
排序：
name age tel
1 Lulu 34 10078
2 Lucy 23 12789
0 Tom 18 10089
'''
3.4 获取行列数据
- df.loc[]
import pandas as pd
import numpy as np
t = pd.DataFrame(np.arange(12).reshape(3, 4), index=list("abc"), columns=list("WXYZ"))
print(t)
# 通过索引获取行列交点，该方法得到的数据是numpy.int32
print(t.loc["a", "W"])
print(type(t.loc["a", "W"]))
# 通过索引获取行列交点，该方法得到的数据是DataFrame对象
print(t.loc[["a"], ["W"]])
print(type(t.loc[["a"], ["W"]]))
# 通过索引获取单行多列（单列多行）,该方法得到的数据是Series对象
print(t.loc["a", ["W", "X"]])
print(type(t.loc["a", ["W", "X"]]))
# 通过索引获取间隔的多行多列，该方法得到的数据是DataFrame对象
print(t.loc[["a","c"],["X","Z"]])
print(type(t.loc[["a","c"],["X","Z"]]))
# 通过索引获取连续的多行多列，该方法得到的数据是DataFrame对象
print(t.loc["a":"c","W":"Z"])
print(type(t.loc["a":"c","W":"Z"]))
'''
W X Y Z
a 0 1 2 3
b 4 5 6 7
c 8 9 10 11
0
W
a 0
W 0
X 1
Name: a, dtype: int32
X Z
a 1 3
c 9 11
W X Y Z
a 0 1 2 3
b 4 5 6 7
c 8 9 10 11
'''
- df.iloc[]
import pandas as pd
import numpy as np
t = pd.DataFrame(np.arange(12).reshape(3, 4), index=list("abc"), columns=list("WXYZ"))
print(t)
# 通过位置获取单行数据，该方法得到的数据是Series对象
print(t.iloc[0])
print(type(t.iloc[0]))
# 通过位置获取行列交点，该方法得到的数据是numpy.int32对象
print(t.iloc[0, 0])
print(type(t.iloc[0, 0]))
# 通过位置获取行列交点，该方法得到的数据是DataFrame对象
print(t.iloc[[0], [0]])
print(type(t.iloc[[0], [0]]))
# 通过位置获取单行多列（单列多行）数据，该方法得到的数据是Series对象
print(t.iloc[0, [0, 1, 2]])
print(type(t.iloc[0, [0, 1, 2]]))
# 通过位置获取间隔的多行多列数据，该方法得到的数据是DataFrame对象
print(t.iloc[[0, 2], [1, 3]])
print(type(t.iloc[[0, 2], [1, 3]]))
# 通过位置获取连续的多行多列数据，该方法得到的数据是DataFrame对象
print(t.iloc[0:2, 0:3])
print(type(t.iloc[0:2, 0:3]))
'''
W X Y Z
a 0 1 2 3
b 4 5 6 7
c 8 9 10 11
W 0
X 1
Y 2
Z 3
Name: a, dtype: int32
0
W
a 0
W 0
X 1
Y 2
Name: a, dtype: int32
X Z
a 1 3
c 9 11
W X Y
a 0 1 2
b 4 5 6
'''
- 布尔索引
import pandas as pd
import numpy as np
t = pd.DataFrame(np.arange(12).reshape(3, 4), index=list("abc"), columns=list("WXYZ"))
print(t)
# 单个条件筛选
print(t[t["W"] > 6])
# 多个条件筛选，条件之间用括号括起来
print(t[(t["W"]>3)&(t["X"]<10)])
'''
W X Y Z
a 0 1 2 3
b 4 5 6 7
c 8 9 10 11
W X Y Z
c 8 9 10 11
W X Y Z
b 4 5 6 7
c 8 9 10 11
'''
4. 数据的合并和分组聚合 4.1 字符串离散化
import pandas as pd
from matplotlib import pyplot as plt
import numpy as np
file_path = "./IMDB-Movie-Data.csv"
df = pd.read_csv(file_path)
print(df["Genre"].head(3))
#统计分类的列表
temp_list = df["Genre"].str.split(",").tolist() #形式如[[],[],[]]
genre_list = list(set([i for j in temp_list for i in j]))
#构造全为0的数组
zeros_df = pd.DataFrame(np.zeros((df.shape[0],len(genre_list))),columns=genre_list)
print(zeros_df)
#给每个电影出现分类的位置赋值1
for i in range(df.shape[0]):
#zeros_df.loc[0,["Sci-fi","Mucical"]] = 1
zeros_df.loc[i,temp_list[i]] = 1
# print(zeros_df.head(3))
#统计每个分类的电影的数量和
genre_count = zeros_df.sum(axis=0)
print(genre_count)
#排序
genre_count = genre_count.sort_values()
_x = genre_count.index
_y = genre_count.values
#画图
plt.figure(figsize=(20,8),dpi=80)
plt.bar(range(len(_x)),_y,width=0.4,color="orange")
plt.xticks(range(len(_x)),_x)
plt.show()
'''
0 Action,Adventure,Sci-Fi
1 Adventure,Mystery,Sci-Fi
2 Horror,Thriller
Name: Genre, dtype: object
Crime Action Fantasy Sport ... Drama Biography Music Romance
0 0.0 0.0 0.0 0.0 ... 0.0 0.0 0.0 0.0
1 0.0 0.0 0.0 0.0 ... 0.0 0.0 0.0 0.0
2 0.0 0.0 0.0 0.0 ... 0.0 0.0 0.0 0.0
3 0.0 0.0 0.0 0.0 ... 0.0 0.0 0.0 0.0
4 0.0 0.0 0.0 0.0 ... 0.0 0.0 0.0 0.0
.. ... ... ... ... ... ... ... ... ...
995 0.0 0.0 0.0 0.0 ... 0.0 0.0 0.0 0.0
996 0.0 0.0 0.0 0.0 ... 0.0 0.0 0.0 0.0
997 0.0 0.0 0.0 0.0 ... 0.0 0.0 0.0 0.0
998 0.0 0.0 0.0 0.0 ... 0.0 0.0 0.0 0.0
999 0.0 0.0 0.0 0.0 ... 0.0 0.0 0.0 0.0
[1000 rows x 20 columns]
Crime 150.0
Action 303.0
Fantasy 101.0
Sport 18.0
Sci-Fi 120.0
Animation 49.0
Western 7.0
War 13.0
History 29.0
Horror 119.0
Comedy 279.0
Thriller 195.0
Adventure 259.0
Mystery 106.0
Family 51.0
Musical 5.0
Drama 513.0
Biography 81.0
Music 16.0
Romance 141.0
dtype: float64
'''
4.2 join() 合并数据
- 将行索引相同的数据合并到一起；
join(self, other, on=None, how='left', lsuffix='', rsuffix='', sort=False) -> 'DataFrame'
'''
Examples
--------
>>> df = pd.DataFrame({'key': ['K0', 'K1', 'K2', 'K3', 'K4', 'K5'],
... 'A': ['A0', 'A1', 'A2', 'A3', 'A4', 'A5']})
>>> df
key A
0 K0 A0
1 K1 A1
2 K2 A2
3 K3 A3
4 K4 A4
5 K5 A5
>>> other = pd.DataFrame({'key': ['K0', 'K1', 'K2'],
... 'B': ['B0', 'B1', 'B2']})
>>> other
key B
0 K0 B0
1 K1 B1
2 K2 B2
Join DataFrames using their indexes.
>>> df.join(other, lsuffix='_caller', rsuffix='_other')
key_caller A key_other B
0 K0 A0 K0 B0
1 K1 A1 K1 B1
2 K2 A2 K2 B2
3 K3 A3 NaN NaN
4 K4 A4 NaN NaN
5 K5 A5 NaN NaN
'''
4.3 merge() 合并数据
- 按照指定的列把数据按照一定的方式合并到一起；
merge(self, right,
how='inner',
on=None,
left_on=None, right_on=None,
left_index=False, right_index=False,
sort=False,
suffixes=('_x', '_y'),
copy=True,
indicator=False,
validate=None) -> 'DataFrame'
'''
Parameters
----------
right : DataFrame or named Series
Object to merge with.
how : {'left', 'right', 'outer', 'inner', 'cross'}, default 'inner'
Type of merge to be performed.
* left: use only keys from left frame, similar to a SQL left outer join;
preserve key order.
* right: use only keys from right frame, similar to a SQL right outer join;
preserve key order.
* outer: use union of keys from both frames, similar to a SQL full outer
join; sort keys lexicographically.
* inner: use intersection of keys from both frames, similar to a SQL inner
join; preserve the order of the left keys.
* cross: creates the cartesian product from both frames, preserves the order of the left keys.
on : label or list
Column or index level names to join on. These must be found in both
DataFrames. If `on` is None and not merging on indexes then this defaults
to the intersection of the columns in both DataFrames.
left_on : label or list, or array-like
Column or index level names to join on in the left DataFrame.
right_on : label or list, or array-like
Column or index level names to join on in the right DataFrame.
left_index : bool, default False
Use the index from the left DataFrame as the join key(s).
right_index : bool, default False
Use the index from the right DataFrame as the join key. Same caveats as
left_index.
sort : bool, default False
Sort the join keys lexicographically in the result DataFrame.
suffixes : list-like, default is ("_x", "_y")
A length-2 sequence where each element is optionally a string
indicating the suffix to add to overlapping column names in
`left` and `right` respectively.
copy : bool, default True
If False, avoid copy if possible.
indicator : bool or str, default False
If True, adds a column to the output DataFrame called "_merge" with
information on the source of each row. The column can be given a different
name by providing a string argument.
validate : str, optional
If specified, checks if merge is of specified type.
'''
4.4 分组 .groupby()
groupby(self, by=None, axis=0, level=None, as_index: 'bool' = True, sort: 'bool' = True, group_keys: 'bool' = True, squeeze: 'bool' =
import pandas as pd
import numpy as np
file_path = "./starbucks_store_worldwide.csv"
df = pd.read_csv(file_path)
# print(df.head(1))
# print(df.info())
grouped = df.groupby(by="Country")
print(grouped)
#DataFrameGroupBy：
#可以进行遍历
for i,j in grouped:
print(i) #打印国家名
print("-"*100)
print(j,type(j)) #打印国家数据
print("*"*100)
print(df[df["Country"]=="US"]) #显示美国数据
#数据按照多个条件进行分组,返回Series grouped = df["Brand"].groupby(by=[df["Country"],df["State/Province"]]).count() print(grouped) print(type(grouped))#虽然是三列，但是结果是Series类型，前两列是索引
#再加一层[],就可以返回DataFrame类型
grouped1 = df[["Brand"]].groupby(by=[df["Country"],df["State/Province"]]).count()
grouped2= df.groupby(by=[df["Country"],df["State/Province"]])[["Brand"]].count()
grouped3 = df.groupby(by=[df["Country"],df["State/Province"]]).count()[["Brand"]]
print(grouped1,type(grouped1))
print("*"*100)
print(grouped2,type(grouped2))
print("*"*100)
print(grouped3,type(grouped3))
4.5 聚合 .count()
count(self, axis=0, level=None, numeric_only=False)
'''
Parameters
----------
axis : {0 or 'index', 1 or 'columns'}, default 0
If 0 or 'index' counts are generated for each column.
If 1 or 'columns' counts are generated for each row.
level : int or str, optional
If the axis is a `MultiIndex` (hierarchical), count along a
particular `level`, collapsing into a `DataFrame`.
A `str` specifies the level name.
numeric_only : bool, default False
Include only `float`, `int` or `boolean` data.
'''
print(grouped["Brand"].count()) country_count = grouped["Brand"].count() print(country_count["US"]) print(country_count["CN"]) ''' Country AD 1 AE 144 AR 108 AT 18 AU 22 ... TT 3 TW 394 US 13608 VN 25 ZA 3 Name: Brand, Length: 73, dtype: int64 13608 2734 '''
5. 索引和复合索引 5.1 简单索引
- 获取index：df.index；
- 指定设置index ：df.index = [‘x’,‘y’]；
- 重新设置index : df.reindex(list(“abcedf”))；
- 指定某一列作为index ：df.set_index(“Country”,drop=False)，drop=True则删掉被用作索引的列；
- 返回index的唯一值：df.set_index(“Country”).index.unique()；
a = pd.DataFrame({'a': range(7),'b': range(7, 0, -1),'c': ['one','one','one','two','two','two', 'two'],'d': list("hjklmno")})
'''
a b c d
0 0 7 one h
1 1 6 one j
2 2 5 one k
3 3 4 two l
4 4 3 two m
5 5 2 two n
6 6 1 two o
'''
b = a.set_index(["c","d"])
'''
a b
c d
one h 0 7
j 1 6
k 2 5
two l 3 4
m 4 3
n 5 2
o 6 1
'''
c = b["a"]
'''
c d
one h 0
j 1
k 2
two l 3
m 4
n 5
o 6
Name: a, dtype: int64
'''
c["one"]["j"] # 1
c["one"]
'''
d
h 0
j 1
k 2
Name: a, dtype: int64
'''
d = a.set_index(["d","c"])["a"]
'''
d c
h one 0
j one 1
k one 2
l two 3
m two 4
n two 5
o two 6
Name: a, dtype: int64
'''
type(d)
''''''
d.index
'''
MultiIndex([('h', 'one'),
('j', 'one'),
('k', 'one'),
('l', 'two'),
('m', 'two'),
('n', 'two'),
('o', 'two')],
names=['d', 'c'])
'''
d.swaplevel()
'''
c d
one h 0
j 1
k 2
two l 3
m 4
n 5
o 6
Name: a, dtype: int64
'''
d.swaplevel()["one"]
'''
d
h 0
j 1
k 2
Name: a, dtype: int64
'''
d.swaplevel()["one"].index
'''Index(['h', 'j', 'k'], dtype='object', name='d')'''
d.swaplevel()["one"]["h"] # 0
d.swaplevel()["one","h"] # 0
5.3 DataFrame复合索引
# DataFrame不能直接索引，需使用loc函数
b
'''
a b
c d
one h 0 7
j 1 6
k 2 5
two l 3 4
m 4 3
n 5 2
o 6 1
'''
b.loc["one"].loc["h"]
'''
a 0
b 7
Name: h, dtype: int64
'''
b.swaplevel().loc["h"]
'''
a b
c
one 0 7
'''
6. pandas 中的时间序列 6.1 基本操作
date_range(start=None,
end=None,
periods=None,
freq=None,
tz=None,
normalize=False,
name=None,
closed=None,
**kwargs) -> pandas.core.indexes.datetimes.DatetimeIndex
'''
Parameters
----------
start : str or datetime-like, optional
Left bound for generating dates.
end : str or datetime-like, optional
Right bound for generating dates.
periods : int, optional
Number of periods to generate.
freq : str or DateOffset, default 'D'
Frequency strings can have multiples, e.g. '5H'. See
:ref:`here ` for a list of
frequency aliases.
tz : str or tzinfo, optional
Time zone name for returning localized DatetimeIndex, for example
'Asia/Hong_Kong'. By default, the resulting DatetimeIndex is
timezone-naive.
normalize : bool, default False
Normalize start/end dates to midnight before generating date range.
name : str, default None
Name of the resulting DatetimeIndex.
closed : {None, 'left', 'right'}, optional
Make the interval closed with respect to the given frequency to
the 'left', 'right', or both sides (None, the default).
**kwargs
For compatibility. Has no effect on the result.
'''
import pandas as pd
time1 = pd.date_range(start="20171230",end="20180131",freq='D')
print("每隔一天的时间索引:n",time1)
print(type(time1))
time2 = pd.date_range(start="20171230",end="20180131",freq='10D')
print("每隔十天的时间索引:n",time2)
time3 = pd.date_range(start="20171230",periods=10,freq='M')
print("按月生成的时间索引:n",time3)
time4 = pd.date_range(start="2017/12/30",periods=10,freq='M')
print("按月生成的时间索引:n",time4)
time5 = pd.date_range(start="2017-12-30 10:10:30",periods=10,freq='H')
print("按小时生成的时间索引:n",time5)
'''
每隔一天的时间索引:
DatetimeIndex(['2017-12-30', '2017-12-31', '2018-01-01', '2018-01-02',
'2018-01-03', '2018-01-04', '2018-01-05', '2018-01-06',
'2018-01-07', '2018-01-08', '2018-01-09', '2018-01-10',
'2018-01-11', '2018-01-12', '2018-01-13', '2018-01-14',
'2018-01-15', '2018-01-16', '2018-01-17', '2018-01-18',
'2018-01-19', '2018-01-20', '2018-01-21', '2018-01-22',
'2018-01-23', '2018-01-24', '2018-01-25', '2018-01-26',
'2018-01-27', '2018-01-28', '2018-01-29', '2018-01-30',
'2018-01-31'],
dtype='datetime64[ns]', freq='D')
每隔十天的时间索引:
DatetimeIndex(['2017-12-30', '2018-01-09',
'2018-01-19', '2018-01-29'],
dtype='datetime64[ns]', freq='10D')
按月生成的时间索引:
DatetimeIndex(['2017-12-31', '2018-01-31',
'2018-02-28', '2018-03-31',
'2018-04-30', '2018-05-31',
'2018-06-30', '2018-07-31',
'2018-08-31', '2018-09-30'],
dtype='datetime64[ns]', freq='M')
按月生成的时间索引:
DatetimeIndex(['2017-12-31', '2018-01-31',
'2018-02-28', '2018-03-31',
'2018-04-30', '2018-05-31',
'2018-06-30', '2018-07-31',
'2018-08-31', '2018-09-30'],
dtype='datetime64[ns]', freq='M')
按小时生成的时间索引:
DatetimeIndex(['2017-12-30 10:10:30',
'2017-12-30 11:10:30',
'2017-12-30 12:10:30',
'2017-12-30 13:10:30',
'2017-12-30 14:10:30',
'2017-12-30 15:10:30',
'2017-12-30 16:10:30',
'2017-12-30 17:10:30',
'2017-12-30 18:10:30',
'2017-12-30 19:10:30'],
dtype='datetime64[ns]', freq='H')
'''
6.2 DataFrame中使用时间序列
import pandas as pd
index = pd.date_range("20170101",periods=10)
df= pd.DataFrame(np.random.rand(10),index=index)
print(df)
'''
2017-01-01 0.759522
2017-01-02 0.203664
2017-01-03 0.682486
2017-01-04 0.757391
2017-01-05 0.017058
2017-01-06 0.777559
2017-01-07 0.012139
2017-01-08 0.248449
2017-01-09 0.231084
2017-01-10 0.565542
'''
6.3 重采样
- 重采样，将时间序列从一个频率转化为另一个频率进行处理的过程；
- 将高频率数据转化为低频率数据为降采样；
- 低频率转化为高频率为升采样；
import pandas as pd
t = pd.DataFrame(np.random.uniform(10,50,(100,1)),index=pd.date_range("20170131",periods=100))
print(t)
# 按月降采样，计算每月的平均值
print("按月降采样，计算每月平均值:n",t.resample("M").mean())
# 按10天降采样，计算天数
print("按10天降采样，计算天数",t.resample("10D").count())
'''
0
2017-01-31 31.818006
2017-02-01 28.205578
2017-02-02 25.054178
2017-02-03 36.538507
2017-02-04 33.518071
... ...
2017-05-06 23.997504
2017-05-07 17.590125
2017-05-08 38.955988
2017-05-09 42.341979
2017-05-10 15.153642
[100 rows x 1 columns]
按月降采样，计算每月平均值:
0
2017-01-31 31.818006
2017-02-28 29.563172
2017-03-31 28.504768
2017-04-30 33.032262
2017-05-31 30.556806
按10天降采样，计算天数
0
2017-01-31 10
2017-02-10 10
2017-02-20 10
2017-03-02 10
2017-03-12 10
2017-03-22 10
2017-04-01 10
2017-04-11 10
2017-04-21 10
2017-05-01 10
'''
6.4 PeriodIndex时间段
periods = pd.PeriodIndex(year=data["year"],month=data["month"],day=data["day"],hour=data["hour"],freq="H")
