diff --git a/module3/exo1/influenza-like-illness-analysis.ipynb b/module3/exo1/influenza-like-illness-analysis.ipynb index d45f16ad9fcd60457f7eaaf5657f00177fdd3523..0f4a8df91fb2e23b8f0bd9baf5a915cdc423b74c 100644 --- a/module3/exo1/influenza-like-illness-analysis.ipynb +++ b/module3/exo1/influenza-like-illness-analysis.ipynb @@ -2479,8 +2479,12 @@ "metadata": {}, "outputs": [], "source": [ - "# Define the name of the local file\n", - "local_data_file = \"incidence-PAY-3.csv\"" + "%matplotlib inline\n", + "import matplotlib.pyplot as plt\n", + "import pandas as pd\n", + "import isoweek\n", + "import os\n", + "import requests\n" ] }, { @@ -2488,13 +2492,45 @@ "execution_count": null, "metadata": {}, "outputs": [], + "source": [ + "# Define the URL of the remote CSV file\n", + "remote_data_url = \"http://www.sentiweb.fr/datasets/incidence-PAY-3.csv\"\n", + "\n", + "# Define the name of the local file\n", + "local_data_file = \"incidence-PAY-3.csv\"\n" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [ + { + "ename": "NameError", + "evalue": "name 'local_data_file' is not defined", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[0;31m# Read the data from the local file into a Pandas DataFrame\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0mraw_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread_csv\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlocal_data_file\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mskiprows\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", + "\u001b[0;31mNameError\u001b[0m: name 'local_data_file' is not defined" + ] + } + ], "source": [ "# Check if the local file exists\n", "if not os.path.isfile(local_data_file):\n", " # If the local file does not exist, download the data from the remote URL\n", " response = requests.get(remote_data_url)\n", " with open(local_data_file, \"w\") as f:\n", - " f.write(response.text)" + " f.write(response.text)\n", + "\n", + "# Read the data from the local file into a Pandas DataFrame\n", + "raw_data = pd.read_csv(local_data_file, skiprows=1)\n", + "\n", + "# Remove rows with missing values\n", + "data = raw_data.dropna().copy()\n", + "\n" ] }, { @@ -2503,8 +2539,50 @@ "metadata": {}, "outputs": [], "source": [ - "# Read the data from the local file into a Pandas DataFrame\n", - "raw_data = pd.read_csv(local_data_file, skiprows=1)" + "# Define a function to convert year and week integers to Pandas Period objects\n", + "def convert_week(year_and_week_int):\n", + " year_and_week_str = str(year_and_week_int)\n", + " year = int(year_and_week_str[:4])\n", + " week = int(year_and_week_str[4:])\n", + " w = isoweek.Week(year, week)\n", + " return pd.Period(w.day(0), 'W')\n", + "\n", + "# Add a column to the DataFrame containing the Period objects\n", + "data['period'] = [convert_week(yw) for yw in data['week']]\n", + "\n", + "# Sort the data by the period column\n", + "sorted_data = data.set_index('period').sort_index()\n", + "\n", + "# Check for gaps in the data and print any that are found\n", + "periods = sorted_data.index\n", + "for p1, p2 in zip(periods[:-1], periods[1:]):\n", + " delta = p2.to_timestamp() - p1.end_time\n", + " if delta > pd.Timedelta('1s'):\n", + " print(p1, p2)\n", + "\n", + "# Plot the incidence data over time\n", + "sorted_data['inc'].plot()\n", + "\n", + "# Plot the last 200 data points\n", + "sorted_data['inc'][-200:].plot()\n", + "\n", + "# Compute the total incidence for each year and plot the results\n", + "first_august_week = [pd.Period(pd.Timestamp(y, 8, 1), 'W')\n", + " for y in range(1985,\n", + " sorted_data.index[-1].year)]\n", + "year = []\n", + "yearly_incidence = []\n", + "for week1, week2 in zip(first_august_week[:-1],\n", + " first_august_week[1:]):\n", + " one_year = sorted_data['inc'][week1:week2-1]\n", + " assert abs(len(one_year)-52) < 2\n", + " yearly_incidence.append(one_year.sum())\n", + " year.append(week2.year)\n", + "yearly_incidence = pd.Series(data=yearly_incidence, index=year)\n", + "\n", + "yearly_incidence.plot(style='*')\n", + "yearly_incidence.sort_values()\n", + "yearly_incidence.hist(xrot=20)\n" ] } ],