{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "29d44422-b6ea-4cdd-8366-7a9daedc8dd4",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "\n",
    "from classes import Tweet\n",
    "from database import connect as connect_db\n",
    "\n",
    "db = connect_db()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "0d043248-c8ad-47bc-90b6-40f02ab279ba",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "got 5190 tweets\n"
     ]
    }
   ],
   "source": [
    "# load in all tweet texts\n",
    "cursor = db.find({'account': '@Maersk'})\n",
    "text_list = list()\n",
    "for element in cursor:\n",
    "    try:\n",
    "        text_list.append(element['text'])\n",
    "    except:\n",
    "        print(f\"failed getting {element['_id']}\")\n",
    "        \n",
    "print(f'got {len(text_list)} tweets')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "d320a5e9-926e-42dd-983e-9cf1b49fdae9",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total characters:  906246\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "\"A spotless deck before delivery Susan Mærsk was built by Odense Steel Shipyard in 1954 and a final inspection of the ship takes place before delivery. The ship was deployed on The Far East service - the company's first liner service initiated in 1928.  #Maerskheritage #Maersk At #Maersk, we are honoured to partner with customers who share our vision for a cleaner, more sustainable future. By spearheading the movement to provide green shipping solutions, we’re building a brighter tomorrow for our world. More: https://bddy.me/3mSdkug #AllTheWay 14 years cooking for a multi-cultural crew!  For Sandy,  a workday onboard of Mette Maersk starts at 5:30 in the morning, as she gets ready to bake a new batch of fresh bread, to be ready in time for breakfast with great joy from the crew onboard. #Maersk #Lifeatsea #Chiefcook Embracing #Equity on #InternationalWomenDay! The transport and logistics industry has traditionally been male dominated, and this is no exception for Maersk. This needs to c\""
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# combine all text into one list\n",
    "full_text = ' '.join(text_list)\n",
    "full_text = full_text.replace('\\n', ' ')\n",
    "print('total characters: ', len(full_text))\n",
    "full_text[:1000]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "2b9f7c25-af69-4862-8ac0-edb87afc0c52",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# clean text\n",
    "from cleantext import clean\n",
    "\n",
    "full_text = clean(\n",
    "    text=full_text,\n",
    "    fix_unicode=True,\n",
    "    to_ascii=True,\n",
    "    lower=True,\n",
    "    normalize_whitespace=True,\n",
    "    no_line_breaks=True,\n",
    "    no_urls=True,\n",
    "    no_emails=True,\n",
    "    no_phone_numbers=True,\n",
    "    no_numbers=True,\n",
    "    no_digits=False,\n",
    "    no_currency_symbols=True,\n",
    "    no_punct=True,\n",
    "    no_emoji=True,\n",
    "    replace_with_url='',\n",
    "    replace_with_email='',\n",
    "    replace_with_phone_number='',\n",
    "    replace_with_number='',\n",
    "    #replace_with_digit='', # to be able to handle CO2\n",
    "    replace_with_currency_symbol='',\n",
    "    replace_with_punct='',\n",
    "    lang='en'\n",
    ")\n",
    "\n",
    "# handle case from CO2 and remove other digits\n",
    "#full_text = full_text.replace(' co<digit> ', ' co2 ')\n",
    "#full_text = full_text.replace('<digit>', '')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "dabcc3c3-7dad-4d12-8e12-7ce3a80305e0",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total words:  80155\n"
     ]
    }
   ],
   "source": [
    "# remove unwanted word classes\n",
    "import nltk\n",
    "#nltk.download('punkt')\n",
    "#nltk.download('averaged_perceptron_tagger')\n",
    "token_list = nltk.word_tokenize(full_text)\n",
    "tag_list = nltk.pos_tag(token_list)\n",
    "\n",
    "unwanted_class_list = [\n",
    "    'CC', # coordinating conjunction\n",
    "    'DT', # determiner\n",
    "    'IN', # preposition/subordinating conjunction\n",
    "    'TO', # to go ‘to’ the store\n",
    "    'PRP', # personal pronoun I, he, she\n",
    "    'PRP$', # possessive pronoun my, his, hers\n",
    "]\n",
    "\n",
    "word_list = list()\n",
    "for word, tag in tag_list:\n",
    "    if tag not in unwanted_class_list:\n",
    "        word_list.append(word)\n",
    "        \n",
    "print('total words: ', len(word_list))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "23862fe3-7cfe-4089-af1c-1eb25b33cfcd",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total unique words:  10925\n"
     ]
    }
   ],
   "source": [
    "# find unique words\n",
    "unique_word_list = list(set(word_list))\n",
    "print('total unique words: ', len(unique_word_list))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "5a55242e-82bd-4e1d-acc7-4ded348f5afd",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# count unique words\n",
    "count_list = list()\n",
    "for unique_word in unique_word_list:\n",
    "    count_list.append(word_list.count(unique_word))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "f325ce44-d296-4458-a443-74958f0465ed",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>word</th>\n",
       "      <th>counts</th>\n",
       "      <th>ratio</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>maersk</td>\n",
       "      <td>3207</td>\n",
       "      <td>0.040010</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>is</td>\n",
       "      <td>1025</td>\n",
       "      <td>0.012788</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>more</td>\n",
       "      <td>989</td>\n",
       "      <td>0.012339</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>here</td>\n",
       "      <td>825</td>\n",
       "      <td>0.010293</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>trade</td>\n",
       "      <td>642</td>\n",
       "      <td>0.008009</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>are</td>\n",
       "      <td>639</td>\n",
       "      <td>0.007972</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>how</td>\n",
       "      <td>634</td>\n",
       "      <td>0.007910</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>logistics</td>\n",
       "      <td>611</td>\n",
       "      <td>0.007623</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>global</td>\n",
       "      <td>467</td>\n",
       "      <td>0.005826</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>new</td>\n",
       "      <td>459</td>\n",
       "      <td>0.005726</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>can</td>\n",
       "      <td>453</td>\n",
       "      <td>0.005652</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>shipping</td>\n",
       "      <td>443</td>\n",
       "      <td>0.005527</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>alltheway</td>\n",
       "      <td>342</td>\n",
       "      <td>0.004267</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>learn</td>\n",
       "      <td>339</td>\n",
       "      <td>0.004229</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>have</td>\n",
       "      <td>314</td>\n",
       "      <td>0.003917</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>be</td>\n",
       "      <td>305</td>\n",
       "      <td>0.003805</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>see</td>\n",
       "      <td>302</td>\n",
       "      <td>0.003768</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>supplychain</td>\n",
       "      <td>296</td>\n",
       "      <td>0.003693</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>supply</td>\n",
       "      <td>295</td>\n",
       "      <td>0.003680</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>will</td>\n",
       "      <td>281</td>\n",
       "      <td>0.003506</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>read</td>\n",
       "      <td>274</td>\n",
       "      <td>0.003418</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>has</td>\n",
       "      <td>271</td>\n",
       "      <td>0.003381</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>what</td>\n",
       "      <td>267</td>\n",
       "      <td>0.003331</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>world</td>\n",
       "      <td>261</td>\n",
       "      <td>0.003256</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>container</td>\n",
       "      <td>248</td>\n",
       "      <td>0.003094</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>business</td>\n",
       "      <td>243</td>\n",
       "      <td>0.003032</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>growth</td>\n",
       "      <td>235</td>\n",
       "      <td>0.002932</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>watch</td>\n",
       "      <td>230</td>\n",
       "      <td>0.002869</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>one</td>\n",
       "      <td>228</td>\n",
       "      <td>0.002844</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>port</td>\n",
       "      <td>221</td>\n",
       "      <td>0.002757</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           word  counts     ratio\n",
       "0        maersk    3207  0.040010\n",
       "1            is    1025  0.012788\n",
       "2          more     989  0.012339\n",
       "3          here     825  0.010293\n",
       "4         trade     642  0.008009\n",
       "5           are     639  0.007972\n",
       "6           how     634  0.007910\n",
       "7     logistics     611  0.007623\n",
       "8        global     467  0.005826\n",
       "9           new     459  0.005726\n",
       "10          can     453  0.005652\n",
       "11     shipping     443  0.005527\n",
       "12    alltheway     342  0.004267\n",
       "13        learn     339  0.004229\n",
       "14         have     314  0.003917\n",
       "15           be     305  0.003805\n",
       "16          see     302  0.003768\n",
       "17  supplychain     296  0.003693\n",
       "18       supply     295  0.003680\n",
       "19         will     281  0.003506\n",
       "20         read     274  0.003418\n",
       "21          has     271  0.003381\n",
       "22         what     267  0.003331\n",
       "23        world     261  0.003256\n",
       "24    container     248  0.003094\n",
       "25     business     243  0.003032\n",
       "26       growth     235  0.002932\n",
       "27        watch     230  0.002869\n",
       "28          one     228  0.002844\n",
       "29         port     221  0.002757"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# present results\n",
    "import pandas as pd\n",
    "word_df = pd.DataFrame.from_dict(\n",
    "    {\n",
    "        'word': unique_word_list,\n",
    "        'counts': count_list\n",
    "    }\n",
    ")\n",
    "word_df = word_df.sort_values(by='counts', ascending=False).reset_index(drop=True)\n",
    "word_df['ratio'] = word_df['counts'] / len(word_list)\n",
    "\n",
    "word_df.head(30)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "363cd061-5de6-442f-96e6-acc4ee4cace4",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# save results\n",
    "word_df['ratio'] = word_df['ratio'].map('{:.5f}'.format) # ensure consistent formatting in csv\n",
    "word_df.to_csv('word_count.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d4802935-9152-40a2-b459-2d30e47948b6",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
