{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "f80d3dd8-29a4-4ecf-a1d9-d691ab23c71e",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>root</th>\n",
       "      <th>mean_ranking</th>\n",
       "      <th>std_ranking</th>\n",
       "      <th>word_matches</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>challeng</td>\n",
       "      <td>406</td>\n",
       "      <td>248</td>\n",
       "      <td>['challenges', 'challenge', 'challenging']</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>SDG</td>\n",
       "      <td>423</td>\n",
       "      <td>0</td>\n",
       "      <td>['sdgs']</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ESG</td>\n",
       "      <td>635</td>\n",
       "      <td>0</td>\n",
       "      <td>['esg']</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>recycling</td>\n",
       "      <td>848</td>\n",
       "      <td>171</td>\n",
       "      <td>['recycling', 'shiprecycling']</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>CSR</td>\n",
       "      <td>1345</td>\n",
       "      <td>0</td>\n",
       "      <td>['csr']</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>reduc</td>\n",
       "      <td>1359</td>\n",
       "      <td>1199</td>\n",
       "      <td>['reduce', 'reducing', 'reduced', 'reduction',...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>planet</td>\n",
       "      <td>1390</td>\n",
       "      <td>0</td>\n",
       "      <td>['planet']</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>sustainab</td>\n",
       "      <td>1516</td>\n",
       "      <td>1825</td>\n",
       "      <td>['sustainability', 'sustainable', 'sustainable...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>future</td>\n",
       "      <td>1626</td>\n",
       "      <td>1540</td>\n",
       "      <td>['future', 'futureproofing']</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>clean</td>\n",
       "      <td>1709</td>\n",
       "      <td>859</td>\n",
       "      <td>['theoceancleanup', 'cleanup', 'clean', 'clean...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>methanol</td>\n",
       "      <td>1755</td>\n",
       "      <td>1110</td>\n",
       "      <td>['methanol', 'emethanol']</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>garbage</td>\n",
       "      <td>1772</td>\n",
       "      <td>511</td>\n",
       "      <td>['garbage', 'greatpacificgarbagepatch']</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>responsib</td>\n",
       "      <td>1840</td>\n",
       "      <td>1093</td>\n",
       "      <td>['responsible', 'responsibility', 'responsibly...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>carb</td>\n",
       "      <td>1848</td>\n",
       "      <td>1396</td>\n",
       "      <td>['decarbonisation', 'carbon', 'decarbonization...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>chang</td>\n",
       "      <td>2054</td>\n",
       "      <td>1563</td>\n",
       "      <td>['change', 'changing', 'climatechange', 'chang...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>ocean</td>\n",
       "      <td>2217</td>\n",
       "      <td>1657</td>\n",
       "      <td>['ocean', 'theoceancleanup', 'oceans', 'oceanp...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>plastic</td>\n",
       "      <td>2354</td>\n",
       "      <td>1587</td>\n",
       "      <td>['plastic', 'oceanplastic', 'plasticwaste', 'p...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>neutral</td>\n",
       "      <td>2384</td>\n",
       "      <td>1886</td>\n",
       "      <td>['neutral', 'carbonneutral', 'neutrality', 'co...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>environment</td>\n",
       "      <td>2537</td>\n",
       "      <td>1736</td>\n",
       "      <td>['environment', 'environmental', 'unenvironmen...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>sulphur</td>\n",
       "      <td>2572</td>\n",
       "      <td>1524</td>\n",
       "      <td>['sulphur', 'lowsulphur']</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>green</td>\n",
       "      <td>2580</td>\n",
       "      <td>1376</td>\n",
       "      <td>['green', 'greenfuels', 'greener', 'greenfuel'...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>emissions</td>\n",
       "      <td>2912</td>\n",
       "      <td>2065</td>\n",
       "      <td>['emissions', 'carbonemissions', 'zeroemissions']</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>climate</td>\n",
       "      <td>2943</td>\n",
       "      <td>1978</td>\n",
       "      <td>['climateaction', 'climate', 'climatechange', ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>zero</td>\n",
       "      <td>2972</td>\n",
       "      <td>1670</td>\n",
       "      <td>['zero', 'netzero', 'zerocarbon', 'zerocarbons...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>eco</td>\n",
       "      <td>3128</td>\n",
       "      <td>1228</td>\n",
       "      <td>['ecosystem', 'eco', 'maerskecodelivery', 'eco...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>mission</td>\n",
       "      <td>3245</td>\n",
       "      <td>1937</td>\n",
       "      <td>['emissions', 'mission', 'carbonemissions', 'e...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>CO2</td>\n",
       "      <td>3279</td>\n",
       "      <td>2165</td>\n",
       "      <td>['co2', 'co2emission', 'co2neutral']</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>bio</td>\n",
       "      <td>3579</td>\n",
       "      <td>1252</td>\n",
       "      <td>['biofuel', 'biofuels', 'biodiversity', 'biohu...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           root  mean_ranking  std_ranking  \\\n",
       "0      challeng           406          248   \n",
       "1           SDG           423            0   \n",
       "2           ESG           635            0   \n",
       "3     recycling           848          171   \n",
       "4           CSR          1345            0   \n",
       "5         reduc          1359         1199   \n",
       "6        planet          1390            0   \n",
       "7     sustainab          1516         1825   \n",
       "8        future          1626         1540   \n",
       "9         clean          1709          859   \n",
       "10     methanol          1755         1110   \n",
       "11      garbage          1772          511   \n",
       "12    responsib          1840         1093   \n",
       "13         carb          1848         1396   \n",
       "14        chang          2054         1563   \n",
       "15        ocean          2217         1657   \n",
       "16      plastic          2354         1587   \n",
       "17      neutral          2384         1886   \n",
       "18  environment          2537         1736   \n",
       "19      sulphur          2572         1524   \n",
       "20        green          2580         1376   \n",
       "21    emissions          2912         2065   \n",
       "22      climate          2943         1978   \n",
       "23         zero          2972         1670   \n",
       "24          eco          3128         1228   \n",
       "25      mission          3245         1937   \n",
       "26          CO2          3279         2165   \n",
       "27          bio          3579         1252   \n",
       "\n",
       "                                         word_matches  \n",
       "0          ['challenges', 'challenge', 'challenging']  \n",
       "1                                            ['sdgs']  \n",
       "2                                             ['esg']  \n",
       "3                      ['recycling', 'shiprecycling']  \n",
       "4                                             ['csr']  \n",
       "5   ['reduce', 'reducing', 'reduced', 'reduction',...  \n",
       "6                                          ['planet']  \n",
       "7   ['sustainability', 'sustainable', 'sustainable...  \n",
       "8                        ['future', 'futureproofing']  \n",
       "9   ['theoceancleanup', 'cleanup', 'clean', 'clean...  \n",
       "10                          ['methanol', 'emethanol']  \n",
       "11            ['garbage', 'greatpacificgarbagepatch']  \n",
       "12  ['responsible', 'responsibility', 'responsibly...  \n",
       "13  ['decarbonisation', 'carbon', 'decarbonization...  \n",
       "14  ['change', 'changing', 'climatechange', 'chang...  \n",
       "15  ['ocean', 'theoceancleanup', 'oceans', 'oceanp...  \n",
       "16  ['plastic', 'oceanplastic', 'plasticwaste', 'p...  \n",
       "17  ['neutral', 'carbonneutral', 'neutrality', 'co...  \n",
       "18  ['environment', 'environmental', 'unenvironmen...  \n",
       "19                          ['sulphur', 'lowsulphur']  \n",
       "20  ['green', 'greenfuels', 'greener', 'greenfuel'...  \n",
       "21  ['emissions', 'carbonemissions', 'zeroemissions']  \n",
       "22  ['climateaction', 'climate', 'climatechange', ...  \n",
       "23  ['zero', 'netzero', 'zerocarbon', 'zerocarbons...  \n",
       "24  ['ecosystem', 'eco', 'maerskecodelivery', 'eco...  \n",
       "25  ['emissions', 'mission', 'carbonemissions', 'e...  \n",
       "26               ['co2', 'co2emission', 'co2neutral']  \n",
       "27  ['biofuel', 'biofuels', 'biodiversity', 'biohu...  "
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# load words related to sustainability\n",
    "import pandas as pd\n",
    "\n",
    "path = 'word_root_ranking.csv'\n",
    "df = pd.read_csv(path, index_col=0)\n",
    "\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "075f141b-16cd-4577-aa5d-97948fda44ee",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['challenges',\n",
       " 'challenge',\n",
       " 'challenging',\n",
       " 'sdgs',\n",
       " 'esg',\n",
       " 'recycling',\n",
       " 'shiprecycling',\n",
       " 'csr',\n",
       " 'reduce',\n",
       " 'reducing']"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# collect words into list\n",
    "\n",
    "related_words_list = list()\n",
    "for row in df['word_matches']:\n",
    "    word_list = row.strip('][').replace(\"'\", '').split(', ')\n",
    "    for word in word_list:\n",
    "        related_words_list.append(word)\n",
    "        \n",
    "related_words_list[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "5f86e61e-8d79-4aa9-871c-f11c62b12293",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "got 1513 sustainability-related tweets\n"
     ]
    }
   ],
   "source": [
    "# fetch all sustainability-related tweets\n",
    "from classes import Tweet\n",
    "from database import connect as connect_db\n",
    "\n",
    "db = connect_db()\n",
    "\n",
    "# generate database cursor\n",
    "cursor = db.find({'account': '@Maersk'})\n",
    "\n",
    "# fetch tweets\n",
    "tweet_list = list()\n",
    "for element in cursor:\n",
    "    try:\n",
    "        elem_text = element['text'].lower()\n",
    "        related_words_present = any([word in elem_text for word in related_words_list])\n",
    "        if not related_words_present:\n",
    "            continue\n",
    "        tweet_list.append(Tweet(**element))\n",
    "    except:\n",
    "        print(f\"failed getting {element['_id']}\")\n",
    "        \n",
    "print(f'got {len(tweet_list)} sustainability-related tweets')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "0fda59a4-9bcd-4f66-9b68-2070b990ba08",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# prepare output folders\n",
    "from pathlib import Path\n",
    "tweet_dir = Path('tweets').resolve()\n",
    "tweet_dir.mkdir(exist_ok=True)\n",
    "media_dir = tweet_dir / 'media'\n",
    "media_dir.mkdir(exist_ok=True)\n",
    "\n",
    "# save tweets\n",
    "for tweet in tweet_list:\n",
    "    time_str = tweet.time.strftime('%Y-%m-%d-%H-%M')\n",
    "    path = tweet_dir / f\"{time_str}.txt\"\n",
    "    media_name_list = [f'{time_str}_img{i+1}.png' for i in range(len(tweet.images))]\n",
    "    # save tweet content\n",
    "    with open(path, 'w') as f:\n",
    "        f.write(f'account: {tweet.account}\\n')\n",
    "        f.write(f'url: {tweet.url}\\n')\n",
    "        f.write(f'text: {tweet.text}\\n')\n",
    "        f.write(f\"images: {','.join(media_name_list)}\\n\")\n",
    "        f.write(f'video: {tweet.video}')\n",
    "    # save images\n",
    "    for image, filename in zip(tweet.images, media_name_list):\n",
    "        image.save(media_dir / filename)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b2fe8c01-e2fd-4d01-9498-613b8b4d1884",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
