airpollution_step1.py

# pre-process data, i.e. remove NA value, consolidate date and time data into Pandas format
# The first step is to consolidate the date-time information into a single date-time so that we can use it as an index in Pandas.
# 18/09/2019, Z Wang

from pandas import read_csv
from datetime import datetime
# load data
def parse(x):
	return datetime.strptime(x, '%Y %m %d %H')
dataset = read_csv('pollution_raw.csv',  parse_dates = [['year', 'month', 'day', 'hour']], index_col=0, date_parser=parse)
# manually delete the first column data ('No')
dataset.drop('No', axis=1, inplace=True)
# manually specify column names
dataset.columns = ['pollution', 'dew', 'temp', 'press', 'wnd_dir', 'wnd_spd', 'snow', 'rain']
dataset.index.name = 'date'
# mark all NA values with 0
dataset['pollution'].fillna(0, inplace=True)
# drop the first 24 hours
dataset = dataset[24:]
# summarize first 5 rows
print(dataset.head(8))
# save to file
dataset.to_csv('pollution.csv')