{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "f80d3dd8-29a4-4ecf-a1d9-d691ab23c71e",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>root</th>\n",
       "      <th>mean_ranking</th>\n",
       "      <th>std_ranking</th>\n",
       "      <th>word_matches</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>challeng</td>\n",
       "      <td>420</td>\n",
       "      <td>264</td>\n",
       "      <td>['challenges', 'challenge', 'challenging']</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>SDG</td>\n",
       "      <td>423</td>\n",
       "      <td>0</td>\n",
       "      <td>['sdgs']</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ESG</td>\n",
       "      <td>636</td>\n",
       "      <td>0</td>\n",
       "      <td>['esg']</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>recycling</td>\n",
       "      <td>860</td>\n",
       "      <td>154</td>\n",
       "      <td>['recycling', 'shiprecycling']</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>reduc</td>\n",
       "      <td>1242</td>\n",
       "      <td>965</td>\n",
       "      <td>['reduce', 'reducing', 'reduced', 'reduction',...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>planet</td>\n",
       "      <td>1295</td>\n",
       "      <td>0</td>\n",
       "      <td>['planet']</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>CSR</td>\n",
       "      <td>1371</td>\n",
       "      <td>0</td>\n",
       "      <td>['csr']</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>sustainab</td>\n",
       "      <td>1527</td>\n",
       "      <td>1873</td>\n",
       "      <td>['sustainability', 'sustainable', 'sustainable...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>clean</td>\n",
       "      <td>1695</td>\n",
       "      <td>837</td>\n",
       "      <td>['theoceancleanup', 'cleanup', 'clean', 'clean...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>methanol</td>\n",
       "      <td>1700</td>\n",
       "      <td>1044</td>\n",
       "      <td>['methanol', 'emethanol']</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>future</td>\n",
       "      <td>1770</td>\n",
       "      <td>1684</td>\n",
       "      <td>['future', 'futureproofing']</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>garbage</td>\n",
       "      <td>1808</td>\n",
       "      <td>612</td>\n",
       "      <td>['garbage', 'greatpacificgarbagepatch']</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>responsib</td>\n",
       "      <td>1889</td>\n",
       "      <td>1135</td>\n",
       "      <td>['responsible', 'responsibility', 'responsibly...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>carb</td>\n",
       "      <td>1952</td>\n",
       "      <td>1526</td>\n",
       "      <td>['decarbonisation', 'carbon', 'decarbonization...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>chang</td>\n",
       "      <td>2148</td>\n",
       "      <td>1667</td>\n",
       "      <td>['change', 'changing', 'climatechange', 'chang...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>ocean</td>\n",
       "      <td>2169</td>\n",
       "      <td>1629</td>\n",
       "      <td>['ocean', 'theoceancleanup', 'oceans', 'oceanp...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>plastic</td>\n",
       "      <td>2320</td>\n",
       "      <td>1490</td>\n",
       "      <td>['plastic', 'oceanplastic', 'plasticwaste', 'p...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>neutral</td>\n",
       "      <td>2382</td>\n",
       "      <td>1798</td>\n",
       "      <td>['neutral', 'carbonneutral', 'neutrality', 'co...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>environment</td>\n",
       "      <td>2391</td>\n",
       "      <td>1574</td>\n",
       "      <td>['environment', 'environmental', 'unenvironmen...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>green</td>\n",
       "      <td>2393</td>\n",
       "      <td>1158</td>\n",
       "      <td>['green', 'greenfuels', 'greener', 'greenfuel'...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>sulphur</td>\n",
       "      <td>2828</td>\n",
       "      <td>1772</td>\n",
       "      <td>['sulphur', 'lowsulphur']</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>emissions</td>\n",
       "      <td>2925</td>\n",
       "      <td>2046</td>\n",
       "      <td>['emissions', 'carbonemissions', 'zeroemissions']</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>zero</td>\n",
       "      <td>2977</td>\n",
       "      <td>1646</td>\n",
       "      <td>['zero', 'netzero', 'zerocarbon', 'zerocarbons...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>climate</td>\n",
       "      <td>3046</td>\n",
       "      <td>2095</td>\n",
       "      <td>['climateaction', 'climate', 'climatechange', ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>eco</td>\n",
       "      <td>3074</td>\n",
       "      <td>1269</td>\n",
       "      <td>['ecosystem', 'eco', 'maerskecodelivery', 'eco...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>mission</td>\n",
       "      <td>3247</td>\n",
       "      <td>1950</td>\n",
       "      <td>['emissions', 'mission', 'eucommission', 'carb...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>CO2</td>\n",
       "      <td>3390</td>\n",
       "      <td>2275</td>\n",
       "      <td>['co2', 'co2neutral', 'co2emission']</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>bio</td>\n",
       "      <td>3687</td>\n",
       "      <td>1274</td>\n",
       "      <td>['biofuel', 'biofuels', 'biohuts', 'biodiversi...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           root  mean_ranking  std_ranking   \n",
       "0      challeng           420          264  \\\n",
       "1           SDG           423            0   \n",
       "2           ESG           636            0   \n",
       "3     recycling           860          154   \n",
       "4         reduc          1242          965   \n",
       "5        planet          1295            0   \n",
       "6           CSR          1371            0   \n",
       "7     sustainab          1527         1873   \n",
       "8         clean          1695          837   \n",
       "9      methanol          1700         1044   \n",
       "10       future          1770         1684   \n",
       "11      garbage          1808          612   \n",
       "12    responsib          1889         1135   \n",
       "13         carb          1952         1526   \n",
       "14        chang          2148         1667   \n",
       "15        ocean          2169         1629   \n",
       "16      plastic          2320         1490   \n",
       "17      neutral          2382         1798   \n",
       "18  environment          2391         1574   \n",
       "19        green          2393         1158   \n",
       "20      sulphur          2828         1772   \n",
       "21    emissions          2925         2046   \n",
       "22         zero          2977         1646   \n",
       "23      climate          3046         2095   \n",
       "24          eco          3074         1269   \n",
       "25      mission          3247         1950   \n",
       "26          CO2          3390         2275   \n",
       "27          bio          3687         1274   \n",
       "\n",
       "                                         word_matches  \n",
       "0          ['challenges', 'challenge', 'challenging']  \n",
       "1                                            ['sdgs']  \n",
       "2                                             ['esg']  \n",
       "3                      ['recycling', 'shiprecycling']  \n",
       "4   ['reduce', 'reducing', 'reduced', 'reduction',...  \n",
       "5                                          ['planet']  \n",
       "6                                             ['csr']  \n",
       "7   ['sustainability', 'sustainable', 'sustainable...  \n",
       "8   ['theoceancleanup', 'cleanup', 'clean', 'clean...  \n",
       "9                           ['methanol', 'emethanol']  \n",
       "10                       ['future', 'futureproofing']  \n",
       "11            ['garbage', 'greatpacificgarbagepatch']  \n",
       "12  ['responsible', 'responsibility', 'responsibly...  \n",
       "13  ['decarbonisation', 'carbon', 'decarbonization...  \n",
       "14  ['change', 'changing', 'climatechange', 'chang...  \n",
       "15  ['ocean', 'theoceancleanup', 'oceans', 'oceanp...  \n",
       "16  ['plastic', 'oceanplastic', 'plasticwaste', 'p...  \n",
       "17  ['neutral', 'carbonneutral', 'neutrality', 'co...  \n",
       "18  ['environment', 'environmental', 'unenvironmen...  \n",
       "19  ['green', 'greenfuels', 'greener', 'greenfuel'...  \n",
       "20                          ['sulphur', 'lowsulphur']  \n",
       "21  ['emissions', 'carbonemissions', 'zeroemissions']  \n",
       "22  ['zero', 'netzero', 'zerocarbon', 'zerocarbons...  \n",
       "23  ['climateaction', 'climate', 'climatechange', ...  \n",
       "24  ['ecosystem', 'eco', 'maerskecodelivery', 'eco...  \n",
       "25  ['emissions', 'mission', 'eucommission', 'carb...  \n",
       "26               ['co2', 'co2neutral', 'co2emission']  \n",
       "27  ['biofuel', 'biofuels', 'biohuts', 'biodiversi...  "
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# load words related to sustainability\n",
    "import pandas as pd\n",
    "\n",
    "path = 'word_root_ranking.csv'\n",
    "df = pd.read_csv(path, index_col=0)\n",
    "\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "075f141b-16cd-4577-aa5d-97948fda44ee",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['challenges',\n",
       " 'challenge',\n",
       " 'challenging',\n",
       " 'sdgs',\n",
       " 'esg',\n",
       " 'recycling',\n",
       " 'shiprecycling',\n",
       " 'reduce',\n",
       " 'reducing',\n",
       " 'reduced']"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# collect words into list\n",
    "\n",
    "related_words_list = list()\n",
    "for row in df['word_matches']:\n",
    "    word_list = row.strip('][').replace(\"'\", '').split(', ')\n",
    "    for word in word_list:\n",
    "        related_words_list.append(word)\n",
    "        \n",
    "related_words_list[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "5f86e61e-8d79-4aa9-871c-f11c62b12293",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "got 1513 sustainability-related tweets\n"
     ]
    }
   ],
   "source": [
    "# fetch all sustainability-related tweets\n",
    "from classes import Tweet\n",
    "from database import connect as connect_db\n",
    "\n",
    "db = connect_db()\n",
    "\n",
    "# generate database cursor\n",
    "cursor = db.find({'account': '@Maersk'})\n",
    "\n",
    "# fetch tweets\n",
    "tweet_list = list()\n",
    "for element in cursor:\n",
    "    try:\n",
    "        elem_text = element['text'].lower()\n",
    "        related_words_present = any([word in elem_text for word in related_words_list])\n",
    "        if not related_words_present:\n",
    "            continue\n",
    "        tweet_list.append(Tweet(**element))\n",
    "    except:\n",
    "        print(f\"failed getting {element['_id']}\")\n",
    "        \n",
    "print(f'got {len(tweet_list)} sustainability-related tweets')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "0fda59a4-9bcd-4f66-9b68-2070b990ba08",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# prepare output folders\n",
    "from pathlib import Path\n",
    "tweet_dir = Path('tweets').resolve()\n",
    "tweet_dir.mkdir(exist_ok=True)\n",
    "media_dir = tweet_dir / 'media'\n",
    "media_dir.mkdir(exist_ok=True)\n",
    "\n",
    "# save tweets\n",
    "for tweet in tweet_list:\n",
    "    time_str = tweet.time.strftime('%Y-%m-%d-%H-%M')\n",
    "    path = tweet_dir / f\"{time_str}.txt\"\n",
    "    media_name_list = [f'{time_str}_img{i+1}.png' for i in range(len(tweet.images))]\n",
    "    # save tweet content\n",
    "    with open(path, 'w') as f:\n",
    "        f.write(f'account: {tweet.account}\\n')\n",
    "        f.write(f'url: {tweet.url}\\n')\n",
    "        f.write(f'text: {tweet.text}\\n')\n",
    "        f.write(f\"images: {','.join(media_name_list)}\\n\")\n",
    "        f.write(f'video: {tweet.video}')\n",
    "    # save images\n",
    "    for image, filename in zip(tweet.images, media_name_list):\n",
    "        image.save(media_dir / filename)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b2fe8c01-e2fd-4d01-9498-613b8b4d1884",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
