{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "29d44422-b6ea-4cdd-8366-7a9daedc8dd4",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "\n",
    "from classes import Tweet\n",
    "from database import connect as connect_db\n",
    "\n",
    "db = connect_db()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "0d043248-c8ad-47bc-90b6-40f02ab279ba",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[6], line 4\u001b[0m\n\u001b[1;32m      2\u001b[0m cursor \u001b[39m=\u001b[39m db\u001b[39m.\u001b[39mfind({\u001b[39m'\u001b[39m\u001b[39maccount\u001b[39m\u001b[39m'\u001b[39m: \u001b[39m'\u001b[39m\u001b[39m@Maersk\u001b[39m\u001b[39m'\u001b[39m})\n\u001b[1;32m      3\u001b[0m text_list \u001b[39m=\u001b[39m \u001b[39mlist\u001b[39m()\n\u001b[0;32m----> 4\u001b[0m \u001b[39mfor\u001b[39;00m element \u001b[39min\u001b[39;00m cursor:\n\u001b[1;32m      5\u001b[0m     \u001b[39mtry\u001b[39;00m:\n\u001b[1;32m      6\u001b[0m         text_list\u001b[39m.\u001b[39mappend(element[\u001b[39m'\u001b[39m\u001b[39mtext\u001b[39m\u001b[39m'\u001b[39m])\n",
      "File \u001b[0;32m~/Desktop/Solveig/venv/lib/python3.8/site-packages/pymongo/cursor.py:1248\u001b[0m, in \u001b[0;36mCursor.next\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m   1246\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m__empty:\n\u001b[1;32m   1247\u001b[0m     \u001b[39mraise\u001b[39;00m \u001b[39mStopIteration\u001b[39;00m\n\u001b[0;32m-> 1248\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mlen\u001b[39m(\u001b[39mself\u001b[39m\u001b[39m.\u001b[39m__data) \u001b[39mor\u001b[39;00m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_refresh():\n\u001b[1;32m   1249\u001b[0m     \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m__data\u001b[39m.\u001b[39mpopleft()\n\u001b[1;32m   1250\u001b[0m \u001b[39melse\u001b[39;00m:\n",
      "File \u001b[0;32m~/Desktop/Solveig/venv/lib/python3.8/site-packages/pymongo/cursor.py:1188\u001b[0m, in \u001b[0;36mCursor._refresh\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m   1173\u001b[0m     \u001b[39m# Exhaust cursors don't send getMore messages.\u001b[39;00m\n\u001b[1;32m   1174\u001b[0m     g \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_getmore_class(\n\u001b[1;32m   1175\u001b[0m         \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m__dbname,\n\u001b[1;32m   1176\u001b[0m         \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m__collname,\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m   1186\u001b[0m         \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m__comment,\n\u001b[1;32m   1187\u001b[0m     )\n\u001b[0;32m-> 1188\u001b[0m     \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m__send_message(g)\n\u001b[1;32m   1190\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39mlen\u001b[39m(\u001b[39mself\u001b[39m\u001b[39m.\u001b[39m__data)\n",
      "File \u001b[0;32m~/Desktop/Solveig/venv/lib/python3.8/site-packages/pymongo/cursor.py:1052\u001b[0m, in \u001b[0;36mCursor.__send_message\u001b[0;34m(self, operation)\u001b[0m\n\u001b[1;32m   1049\u001b[0m     \u001b[39mraise\u001b[39;00m InvalidOperation(\u001b[39m\"\u001b[39m\u001b[39mexhaust cursors do not support auto encryption\u001b[39m\u001b[39m\"\u001b[39m)\n\u001b[1;32m   1051\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[0;32m-> 1052\u001b[0m     response \u001b[39m=\u001b[39m client\u001b[39m.\u001b[39;49m_run_operation(\n\u001b[1;32m   1053\u001b[0m         operation, \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_unpack_response, address\u001b[39m=\u001b[39;49m\u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m__address\n\u001b[1;32m   1054\u001b[0m     )\n\u001b[1;32m   1055\u001b[0m \u001b[39mexcept\u001b[39;00m OperationFailure \u001b[39mas\u001b[39;00m exc:\n\u001b[1;32m   1056\u001b[0m     \u001b[39mif\u001b[39;00m exc\u001b[39m.\u001b[39mcode \u001b[39min\u001b[39;00m _CURSOR_CLOSED_ERRORS \u001b[39mor\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m__exhaust:\n\u001b[1;32m   1057\u001b[0m         \u001b[39m# Don't send killCursors because the cursor is already closed.\u001b[39;00m\n",
      "File \u001b[0;32m~/Desktop/Solveig/venv/lib/python3.8/site-packages/pymongo/_csot.py:105\u001b[0m, in \u001b[0;36mapply.<locals>.csot_wrapper\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m    103\u001b[0m         \u001b[39mwith\u001b[39;00m _TimeoutContext(timeout):\n\u001b[1;32m    104\u001b[0m             \u001b[39mreturn\u001b[39;00m func(\u001b[39mself\u001b[39m, \u001b[39m*\u001b[39margs, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs)\n\u001b[0;32m--> 105\u001b[0m \u001b[39mreturn\u001b[39;00m func(\u001b[39mself\u001b[39;49m, \u001b[39m*\u001b[39;49margs, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n",
      "File \u001b[0;32m~/Desktop/Solveig/venv/lib/python3.8/site-packages/pymongo/mongo_client.py:1330\u001b[0m, in \u001b[0;36mMongoClient._run_operation\u001b[0;34m(self, operation, unpack_res, address)\u001b[0m\n\u001b[1;32m   1325\u001b[0m     operation\u001b[39m.\u001b[39mreset()  \u001b[39m# Reset op in case of retry.\u001b[39;00m\n\u001b[1;32m   1326\u001b[0m     \u001b[39mreturn\u001b[39;00m server\u001b[39m.\u001b[39mrun_operation(\n\u001b[1;32m   1327\u001b[0m         sock_info, operation, read_preference, \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_event_listeners, unpack_res\n\u001b[1;32m   1328\u001b[0m     )\n\u001b[0;32m-> 1330\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_retryable_read(\n\u001b[1;32m   1331\u001b[0m     _cmd,\n\u001b[1;32m   1332\u001b[0m     operation\u001b[39m.\u001b[39;49mread_preference,\n\u001b[1;32m   1333\u001b[0m     operation\u001b[39m.\u001b[39;49msession,\n\u001b[1;32m   1334\u001b[0m     address\u001b[39m=\u001b[39;49maddress,\n\u001b[1;32m   1335\u001b[0m     retryable\u001b[39m=\u001b[39;49m\u001b[39misinstance\u001b[39;49m(operation, message\u001b[39m.\u001b[39;49m_Query),\n\u001b[1;32m   1336\u001b[0m )\n",
      "File \u001b[0;32m~/Desktop/Solveig/venv/lib/python3.8/site-packages/pymongo/_csot.py:105\u001b[0m, in \u001b[0;36mapply.<locals>.csot_wrapper\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m    103\u001b[0m         \u001b[39mwith\u001b[39;00m _TimeoutContext(timeout):\n\u001b[1;32m    104\u001b[0m             \u001b[39mreturn\u001b[39;00m func(\u001b[39mself\u001b[39m, \u001b[39m*\u001b[39margs, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs)\n\u001b[0;32m--> 105\u001b[0m \u001b[39mreturn\u001b[39;00m func(\u001b[39mself\u001b[39;49m, \u001b[39m*\u001b[39;49margs, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n",
      "File \u001b[0;32m~/Desktop/Solveig/venv/lib/python3.8/site-packages/pymongo/mongo_client.py:1448\u001b[0m, in \u001b[0;36mMongoClient._retryable_read\u001b[0;34m(self, func, read_pref, session, address, retryable)\u001b[0m\n\u001b[1;32m   1446\u001b[0m             \u001b[39massert\u001b[39;00m last_error \u001b[39mis\u001b[39;00m \u001b[39mnot\u001b[39;00m \u001b[39mNone\u001b[39;00m\n\u001b[1;32m   1447\u001b[0m             \u001b[39mraise\u001b[39;00m last_error\n\u001b[0;32m-> 1448\u001b[0m         \u001b[39mreturn\u001b[39;00m func(session, server, sock_info, read_pref)\n\u001b[1;32m   1449\u001b[0m \u001b[39mexcept\u001b[39;00m ServerSelectionTimeoutError:\n\u001b[1;32m   1450\u001b[0m     \u001b[39mif\u001b[39;00m retrying:\n\u001b[1;32m   1451\u001b[0m         \u001b[39m# The application may think the write was never attempted\u001b[39;00m\n\u001b[1;32m   1452\u001b[0m         \u001b[39m# if we raise ServerSelectionTimeoutError on the retry\u001b[39;00m\n\u001b[1;32m   1453\u001b[0m         \u001b[39m# attempt. Raise the original exception instead.\u001b[39;00m\n",
      "File \u001b[0;32m~/Desktop/Solveig/venv/lib/python3.8/site-packages/pymongo/mongo_client.py:1326\u001b[0m, in \u001b[0;36mMongoClient._run_operation.<locals>._cmd\u001b[0;34m(session, server, sock_info, read_preference)\u001b[0m\n\u001b[1;32m   1324\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39m_cmd\u001b[39m(session, server, sock_info, read_preference):\n\u001b[1;32m   1325\u001b[0m     operation\u001b[39m.\u001b[39mreset()  \u001b[39m# Reset op in case of retry.\u001b[39;00m\n\u001b[0;32m-> 1326\u001b[0m     \u001b[39mreturn\u001b[39;00m server\u001b[39m.\u001b[39;49mrun_operation(\n\u001b[1;32m   1327\u001b[0m         sock_info, operation, read_preference, \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_event_listeners, unpack_res\n\u001b[1;32m   1328\u001b[0m     )\n",
      "File \u001b[0;32m~/Desktop/Solveig/venv/lib/python3.8/site-packages/pymongo/server.py:115\u001b[0m, in \u001b[0;36mServer.run_operation\u001b[0;34m(self, sock_info, operation, read_preference, listeners, unpack_res)\u001b[0m\n\u001b[1;32m    113\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[1;32m    114\u001b[0m     sock_info\u001b[39m.\u001b[39msend_message(data, max_doc_size)\n\u001b[0;32m--> 115\u001b[0m     reply \u001b[39m=\u001b[39m sock_info\u001b[39m.\u001b[39;49mreceive_message(request_id)\n\u001b[1;32m    117\u001b[0m \u001b[39m# Unpack and check for command errors.\u001b[39;00m\n\u001b[1;32m    118\u001b[0m \u001b[39mif\u001b[39;00m use_cmd:\n",
      "File \u001b[0;32m~/Desktop/Solveig/venv/lib/python3.8/site-packages/pymongo/pool.py:821\u001b[0m, in \u001b[0;36mSocketInfo.receive_message\u001b[0;34m(self, request_id)\u001b[0m\n\u001b[1;32m    819\u001b[0m     \u001b[39mreturn\u001b[39;00m receive_message(\u001b[39mself\u001b[39m, request_id, \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mmax_message_size)\n\u001b[1;32m    820\u001b[0m \u001b[39mexcept\u001b[39;00m \u001b[39mBaseException\u001b[39;00m \u001b[39mas\u001b[39;00m error:\n\u001b[0;32m--> 821\u001b[0m     \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_raise_connection_failure(error)\n",
      "File \u001b[0;32m~/Desktop/Solveig/venv/lib/python3.8/site-packages/pymongo/pool.py:819\u001b[0m, in \u001b[0;36mSocketInfo.receive_message\u001b[0;34m(self, request_id)\u001b[0m\n\u001b[1;32m    814\u001b[0m \u001b[39m\u001b[39m\u001b[39m\"\"\"Receive a raw BSON message or raise ConnectionFailure.\u001b[39;00m\n\u001b[1;32m    815\u001b[0m \n\u001b[1;32m    816\u001b[0m \u001b[39mIf any exception is raised, the socket is closed.\u001b[39;00m\n\u001b[1;32m    817\u001b[0m \u001b[39m\"\"\"\u001b[39;00m\n\u001b[1;32m    818\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[0;32m--> 819\u001b[0m     \u001b[39mreturn\u001b[39;00m receive_message(\u001b[39mself\u001b[39;49m, request_id, \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mmax_message_size)\n\u001b[1;32m    820\u001b[0m \u001b[39mexcept\u001b[39;00m \u001b[39mBaseException\u001b[39;00m \u001b[39mas\u001b[39;00m error:\n\u001b[1;32m    821\u001b[0m     \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_raise_connection_failure(error)\n",
      "File \u001b[0;32m~/Desktop/Solveig/venv/lib/python3.8/site-packages/pymongo/network.py:238\u001b[0m, in \u001b[0;36mreceive_message\u001b[0;34m(sock_info, request_id, max_message_size)\u001b[0m\n\u001b[1;32m    236\u001b[0m     data \u001b[39m=\u001b[39m decompress(_receive_data_on_socket(sock_info, length \u001b[39m-\u001b[39m \u001b[39m25\u001b[39m, deadline), compressor_id)\n\u001b[1;32m    237\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[0;32m--> 238\u001b[0m     data \u001b[39m=\u001b[39m _receive_data_on_socket(sock_info, length \u001b[39m-\u001b[39;49m \u001b[39m16\u001b[39;49m, deadline)\n\u001b[1;32m    240\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[1;32m    241\u001b[0m     unpack_reply \u001b[39m=\u001b[39m _UNPACK_REPLY[op_code]\n",
      "File \u001b[0;32m~/Desktop/Solveig/venv/lib/python3.8/site-packages/pymongo/network.py:299\u001b[0m, in \u001b[0;36m_receive_data_on_socket\u001b[0;34m(sock_info, length, deadline)\u001b[0m\n\u001b[1;32m    297\u001b[0m     \u001b[39mif\u001b[39;00m _csot\u001b[39m.\u001b[39mget_timeout():\n\u001b[1;32m    298\u001b[0m         sock_info\u001b[39m.\u001b[39mset_socket_timeout(\u001b[39mmax\u001b[39m(deadline \u001b[39m-\u001b[39m time\u001b[39m.\u001b[39mmonotonic(), \u001b[39m0\u001b[39m))\n\u001b[0;32m--> 299\u001b[0m     chunk_length \u001b[39m=\u001b[39m sock_info\u001b[39m.\u001b[39;49msock\u001b[39m.\u001b[39;49mrecv_into(mv[bytes_read:])\n\u001b[1;32m    300\u001b[0m \u001b[39mexcept\u001b[39;00m BLOCKING_IO_ERRORS:\n\u001b[1;32m    301\u001b[0m     \u001b[39mraise\u001b[39;00m socket\u001b[39m.\u001b[39mtimeout(\u001b[39m\"\u001b[39m\u001b[39mtimed out\u001b[39m\u001b[39m\"\u001b[39m)\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "# load in all tweet texts\n",
    "cursor = db.find({'account': '@Maersk'})\n",
    "text_list = list()\n",
    "for element in cursor:\n",
    "    try:\n",
    "        text_list.append(element['text'])\n",
    "    except:\n",
    "        print(f\"failed getting {element['_id']}\")\n",
    "        \n",
    "print(f'got {len(text_list)} tweets')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9968ba82-9215-4069-be32-b3afa96ebe8e",
   "metadata": {},
   "outputs": [],
   "source": [
    "Questions:\n",
    "‘challengessee’\n",
    "‘challengeor’\n",
    "‘changetheworld’\n",
    "‘changeat’\n",
    "‘oceanswe’\n",
    "‘planets’\n",
    "‘futureproofing’\n",
    "‘leadthefuture’\n",
    "‘futureready’\n",
    "‘futureproof’\n",
    "'futureofshipping' # hashtag\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "23ba20c5-adf4-4655-a5b0-3662956ff9aa",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "for text in text_list:\n",
    "    if 'challengessee' in text:\n",
    "        print(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "d320a5e9-926e-42dd-983e-9cf1b49fdae9",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total characters:  906246\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "\"A spotless deck before delivery Susan Mærsk was built by Odense Steel Shipyard in 1954 and a final inspection of the ship takes place before delivery. The ship was deployed on The Far East service - the company's first liner service initiated in 1928.  #Maerskheritage #Maersk At #Maersk, we are honoured to partner with customers who share our vision for a cleaner, more sustainable future. By spearheading the movement to provide green shipping solutions, we’re building a brighter tomorrow for our world. More: https://bddy.me/3mSdkug #AllTheWay 14 years cooking for a multi-cultural crew!  For Sandy,  a workday onboard of Mette Maersk starts at 5:30 in the morning, as she gets ready to bake a new batch of fresh bread, to be ready in time for breakfast with great joy from the crew onboard. #Maersk #Lifeatsea #Chiefcook Embracing #Equity on #InternationalWomenDay! The transport and logistics industry has traditionally been male dominated, and this is no exception for Maersk. This needs to c\""
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# combine all text into one list\n",
    "full_text = ' '.join(text_list)\n",
    "full_text = full_text.replace('\\n', ' ')\n",
    "print('total characters: ', len(full_text))\n",
    "full_text[:1000]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "918694e3-7ef7-4902-9d78-39a23e531c67",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import re\n",
    "\n",
    "len(re.findall('challengeor', full_text))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "bb70eefd-ee98-43b5-b2f9-5cc4b05ffcd8",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "53"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import re\n",
    "\n",
    "len(re.findall(' CO2', full_text))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "2b9f7c25-af69-4862-8ac0-edb87afc0c52",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# clean text\n",
    "from cleantext import clean\n",
    "\n",
    "full_text = clean(\n",
    "    text=full_text,\n",
    "    fix_unicode=True,\n",
    "    to_ascii=True,\n",
    "    lower=True,\n",
    "    normalize_whitespace=True,\n",
    "    no_line_breaks=True,\n",
    "    no_urls=True,\n",
    "    no_emails=True,\n",
    "    no_phone_numbers=True,\n",
    "    no_numbers=True,\n",
    "    no_digits=False,\n",
    "    no_currency_symbols=True,\n",
    "    no_punct=True,\n",
    "    no_emoji=True,\n",
    "    replace_with_url='',\n",
    "    replace_with_email='',\n",
    "    replace_with_phone_number='',\n",
    "    replace_with_number='',\n",
    "    #replace_with_digit='', # to be able to handle CO2\n",
    "    replace_with_currency_symbol='',\n",
    "    replace_with_punct='',\n",
    "    lang='en'\n",
    ")\n",
    "\n",
    "# handle case from CO2 and remove other digits\n",
    "#full_text = full_text.replace(' co<digit> ', ' co2 ')\n",
    "#full_text = full_text.replace('<digit>', '')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "284be43e-fd23-4f12-a88e-9a869ab480ea",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['co2neutral',\n",
       " 'co2emission',\n",
       " 'co2emission',\n",
       " 'co1prbwlw',\n",
       " 'co1idd6st',\n",
       " 'co1l7bctb',\n",
       " 'co1lfltua',\n",
       " 'co1o1ksg7',\n",
       " 'co1ccchqp',\n",
       " 'co2emissions',\n",
       " 'co11bdxbv',\n",
       " 'co2neutral']"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import re\n",
    "\n",
    "re.findall('co[1-9]\\w+', full_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "f7920c88-6f3c-409c-9f8a-289268c04995",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[]"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import re\n",
    "\n",
    "re.findall(' CO2 ', full_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "527cccef-e9c8-4679-b6ef-6ae09d470353",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "53"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import re\n",
    "\n",
    "len(re.findall(' co2 ', full_text))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "dabcc3c3-7dad-4d12-8e12-7ce3a80305e0",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total words:  80155\n"
     ]
    }
   ],
   "source": [
    "# remove unwanted word classes\n",
    "import nltk\n",
    "#nltk.download('punkt')\n",
    "#nltk.download('averaged_perceptron_tagger')\n",
    "token_list = nltk.word_tokenize(full_text)\n",
    "tag_list = nltk.pos_tag(token_list)\n",
    "\n",
    "unwanted_class_list = [\n",
    "    'CC', # coordinating conjunction\n",
    "    'DT', # determiner\n",
    "    'IN', # preposition/subordinating conjunction\n",
    "    'TO', # to go ‘to’ the store\n",
    "    'PRP', # personal pronoun I, he, she\n",
    "    'PRP$', # possessive pronoun my, his, hers\n",
    "]\n",
    "\n",
    "word_list = list()\n",
    "for word, tag in tag_list:\n",
    "    if tag not in unwanted_class_list:\n",
    "        word_list.append(word)\n",
    "        \n",
    "print('total words: ', len(word_list))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "23862fe3-7cfe-4089-af1c-1eb25b33cfcd",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total unique words:  10925\n"
     ]
    }
   ],
   "source": [
    "# find unique words\n",
    "unique_word_list = list(set(word_list))\n",
    "print('total unique words: ', len(unique_word_list))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "5a55242e-82bd-4e1d-acc7-4ded348f5afd",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# count unique words\n",
    "count_list = list()\n",
    "for unique_word in unique_word_list:\n",
    "    count_list.append(word_list.count(unique_word))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "f325ce44-d296-4458-a443-74958f0465ed",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>word</th>\n",
       "      <th>counts</th>\n",
       "      <th>ratio</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>maersk</td>\n",
       "      <td>3207</td>\n",
       "      <td>0.040010</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>is</td>\n",
       "      <td>1025</td>\n",
       "      <td>0.012788</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>more</td>\n",
       "      <td>989</td>\n",
       "      <td>0.012339</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>here</td>\n",
       "      <td>825</td>\n",
       "      <td>0.010293</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>trade</td>\n",
       "      <td>642</td>\n",
       "      <td>0.008009</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>are</td>\n",
       "      <td>639</td>\n",
       "      <td>0.007972</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>how</td>\n",
       "      <td>634</td>\n",
       "      <td>0.007910</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>logistics</td>\n",
       "      <td>611</td>\n",
       "      <td>0.007623</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>global</td>\n",
       "      <td>467</td>\n",
       "      <td>0.005826</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>new</td>\n",
       "      <td>459</td>\n",
       "      <td>0.005726</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>can</td>\n",
       "      <td>453</td>\n",
       "      <td>0.005652</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>shipping</td>\n",
       "      <td>443</td>\n",
       "      <td>0.005527</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>alltheway</td>\n",
       "      <td>342</td>\n",
       "      <td>0.004267</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>learn</td>\n",
       "      <td>339</td>\n",
       "      <td>0.004229</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>have</td>\n",
       "      <td>314</td>\n",
       "      <td>0.003917</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>be</td>\n",
       "      <td>305</td>\n",
       "      <td>0.003805</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>see</td>\n",
       "      <td>302</td>\n",
       "      <td>0.003768</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>supplychain</td>\n",
       "      <td>296</td>\n",
       "      <td>0.003693</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>supply</td>\n",
       "      <td>295</td>\n",
       "      <td>0.003680</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>will</td>\n",
       "      <td>281</td>\n",
       "      <td>0.003506</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>read</td>\n",
       "      <td>274</td>\n",
       "      <td>0.003418</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>has</td>\n",
       "      <td>271</td>\n",
       "      <td>0.003381</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>what</td>\n",
       "      <td>267</td>\n",
       "      <td>0.003331</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>world</td>\n",
       "      <td>261</td>\n",
       "      <td>0.003256</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>container</td>\n",
       "      <td>248</td>\n",
       "      <td>0.003094</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>business</td>\n",
       "      <td>243</td>\n",
       "      <td>0.003032</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>growth</td>\n",
       "      <td>235</td>\n",
       "      <td>0.002932</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>watch</td>\n",
       "      <td>230</td>\n",
       "      <td>0.002869</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>one</td>\n",
       "      <td>228</td>\n",
       "      <td>0.002844</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>up</td>\n",
       "      <td>221</td>\n",
       "      <td>0.002757</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           word  counts     ratio\n",
       "0        maersk    3207  0.040010\n",
       "1            is    1025  0.012788\n",
       "2          more     989  0.012339\n",
       "3          here     825  0.010293\n",
       "4         trade     642  0.008009\n",
       "5           are     639  0.007972\n",
       "6           how     634  0.007910\n",
       "7     logistics     611  0.007623\n",
       "8        global     467  0.005826\n",
       "9           new     459  0.005726\n",
       "10          can     453  0.005652\n",
       "11     shipping     443  0.005527\n",
       "12    alltheway     342  0.004267\n",
       "13        learn     339  0.004229\n",
       "14         have     314  0.003917\n",
       "15           be     305  0.003805\n",
       "16          see     302  0.003768\n",
       "17  supplychain     296  0.003693\n",
       "18       supply     295  0.003680\n",
       "19         will     281  0.003506\n",
       "20         read     274  0.003418\n",
       "21          has     271  0.003381\n",
       "22         what     267  0.003331\n",
       "23        world     261  0.003256\n",
       "24    container     248  0.003094\n",
       "25     business     243  0.003032\n",
       "26       growth     235  0.002932\n",
       "27        watch     230  0.002869\n",
       "28          one     228  0.002844\n",
       "29           up     221  0.002757"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# present results\n",
    "import pandas as pd\n",
    "word_df = pd.DataFrame.from_dict(\n",
    "    {\n",
    "        'word': unique_word_list,\n",
    "        'counts': count_list\n",
    "    }\n",
    ")\n",
    "word_df = word_df.sort_values(by='counts', ascending=False).reset_index(drop=True)\n",
    "word_df['ratio'] = word_df['counts'] / len(word_list)\n",
    "\n",
    "word_df.head(30)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "363cd061-5de6-442f-96e6-acc4ee4cace4",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# save results\n",
    "word_df['ratio'] = word_df['ratio'].map('{:.5f}'.format) # ensure consistent formatting in csv\n",
    "word_df.to_csv('word_count.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d4802935-9152-40a2-b459-2d30e47948b6",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
