Skip to content
Permalink
main
Switch branches/tags

Name already in use

A tag already exists with the provided branch name. Many Git commands accept both tag and branch names, so creating this branch may cause unexpected behavior. Are you sure you want to create this branch?

7071CEM-CW/My_Crawler.ipynb

Go to file
 
 
Cannot retrieve contributors at this time
1099 lines (1099 sloc) 37.3 KB
Raw Blame
Open in GitHub Desktop
{
"cells": [
{
"cell_type": "code",
"execution_count": 2,
"id": "8f479220",
"metadata": {},
"outputs": [],
"source": [
"import requests\n",
"from bs4 import BeautifulSoup\n",
"import re\n",
"import pandas as pd"
]
},
{
"cell_type": "code",
"execution_count": 26,
"id": "e2ed62e0",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"https://pureportal.coventry.ac.uk/en/organisations/faculty-of-engineering-environment-computing/persons/\n"
]
}
],
"source": [
"#Starting Url\n",
"\n",
"res= requests.get(url)\n",
"soup= BeautifulSoup(res.text,'html.parser')\n",
"pr=soup.find_all('a',class_= 'portal_link btn-primary btn-large')[3]['href']\n",
"new_url=\"https://pureportal.coventry.ac.uk\"+str(pr)\n",
"url= new_url\n",
"res= requests.get(new_url)\n",
"soup= BeautifulSoup(res.text,'html.parser')\n",
"print(url)"
]
},
{
"cell_type": "code",
"execution_count": 19,
"id": "0f65234f",
"metadata": {},
"outputs": [],
"source": [
"urls= [\"https://pureportal.coventry.ac.uk/en/organisations/school-of-mechanical-aerospace-and-automotive-engineering/publications/\",\n",
" \"https://pureportal.coventry.ac.uk/en/organisations/school-of-mechanical-aerospace-and-automotive-engineering/publications/?page=1\"]\n",
"name = []\n",
"dates = []\n",
"authors = []\n",
"\n",
"for url in urls: \n",
" res= requests.get(url)\n",
" soup= BeautifulSoup(res.text,'html.parser')\n",
" for research in soup.find_all('h3',{'class': 'title'}):\n",
" name.append(research.a.span.get_text())\n",
" name\n",
"\n",
" for date in soup.find_all('span',{'class':'date'}):\n",
" dates.append(date.get_text())\n",
" dates\n",
"\n",
" for author in soup.find_all(attrs={'rel':'Person'}):\n",
" authors.append(author.span.get_text())\n",
" authors"
]
},
{
"cell_type": "code",
"execution_count": 23,
"id": "a33e46a9",
"metadata": {},
"outputs": [],
"source": [
"df_new = pd.DataFrame(list(zip(authors,dates,name)),columns =['Authors', 'Dates', 'Publications'])"
]
},
{
"cell_type": "code",
"execution_count": 24,
"id": "d6a647be",
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>Authors</th>\n",
" <th>Dates</th>\n",
" <th>Publications</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Kulkarni, S.</td>\n",
" <td>16 Nov 2021</td>\n",
" <td>A bibliometric review on the implications of r...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>Enebuse, I.</td>\n",
" <td>2021</td>\n",
" <td>A comparative review of hand-eye calibration t...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>Ibrahim, B. S. K. K.</td>\n",
" <td>2021</td>\n",
" <td>Acoustic radiation modes and modal criterion d...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>Manarikkal, I.</td>\n",
" <td>29 Sep 2021</td>\n",
" <td>A critique of the THUMS lower limb model for p...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>Elasha, F.</td>\n",
" <td>1 Mar 2021</td>\n",
" <td>Activation of nano kaolin clay for bio-glycero...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>...</th>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>95</th>\n",
" <td>Cai, R.</td>\n",
" <td>2 Mar 2021</td>\n",
" <td>Part load operation of natural gas fired power...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>96</th>\n",
" <td>Raja Ahsan Shah, R. M.</td>\n",
" <td>2 Mar 2021</td>\n",
" <td>Photovoltaic module efficiency evaluation: The...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>97</th>\n",
" <td>Al Qubeissi, M.</td>\n",
" <td>Nov 2021</td>\n",
" <td>Potential of waste cooking oil biodiesel as re...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>98</th>\n",
" <td>Shah, R. M. R. A.</td>\n",
" <td>25 May 2021</td>\n",
" <td>Process analysis of improved process modificat...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>99</th>\n",
" <td>Qubeissi, M. A.</td>\n",
" <td>24 Nov 2021</td>\n",
" <td>Reducing energy consumption and pollution in t...</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"<p>100 rows × 3 columns</p>\n",
"</div>"
],
"text/plain": [
" Authors Dates \\\n",
"0 Kulkarni, S. 16 Nov 2021 \n",
"1 Enebuse, I. 2021 \n",
"2 Ibrahim, B. S. K. K. 2021 \n",
"3 Manarikkal, I. 29 Sep 2021 \n",
"4 Elasha, F. 1 Mar 2021 \n",
".. ... ... \n",
"95 Cai, R. 2 Mar 2021 \n",
"96 Raja Ahsan Shah, R. M. 2 Mar 2021 \n",
"97 Al Qubeissi, M. Nov 2021 \n",
"98 Shah, R. M. R. A. 25 May 2021 \n",
"99 Qubeissi, M. A. 24 Nov 2021 \n",
"\n",
" Publications \n",
"0 A bibliometric review on the implications of r... \n",
"1 A comparative review of hand-eye calibration t... \n",
"2 Acoustic radiation modes and modal criterion d... \n",
"3 A critique of the THUMS lower limb model for p... \n",
"4 Activation of nano kaolin clay for bio-glycero... \n",
".. ... \n",
"95 Part load operation of natural gas fired power... \n",
"96 Photovoltaic module efficiency evaluation: The... \n",
"97 Potential of waste cooking oil biodiesel as re... \n",
"98 Process analysis of improved process modificat... \n",
"99 Reducing energy consumption and pollution in t... \n",
"\n",
"[100 rows x 3 columns]"
]
},
"execution_count": 24,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"df_new"
]
},
{
"cell_type": "code",
"execution_count": 27,
"id": "f30d776f",
"metadata": {},
"outputs": [],
"source": [
"url= \"https://pureportal.coventry.ac.uk/en/organisations/school-of-mechanical-aerospace-and-automotive-engineering/publications/\"\n",
"name = []\n",
"link = []\n",
"title = []\n",
"\n",
"\n",
"pages=[1,2,3,4,5,6]\n",
"res= requests.get(url)\n",
"soup= BeautifulSoup(res.text,'html.parser')\n",
"for research in soup.find_all('div',{'class': 'title'}):\n",
" name.append(research.a.span.get_text())\n",
" \n",
" link.append(research.div.h3.a['href']) \n",
"\n",
"for i in pages:\n",
" new_url=url+\"?page={}\".format(i)\n",
" res= requests.get(new_url)\n",
" soup= BeautifulSoup(res.text,'html.parser')\n",
" \n",
" for research in soup.find_all('div',{'class': 'result-container'}):\n",
" \n",
" name.append(research.div.h3.a.get_text())\n",
" link.append(research.div.h3.a['href']) \n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "303b5fd6",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": 28,
"id": "555614bd",
"metadata": {},
"outputs": [],
"source": [
"#boss4\n",
"#Web-crawler for all profiles of Faculty of EEC\n",
"Name2 = []\n",
"Research_title2 = []\n",
"Link2 = []\n",
"\n",
"\n",
"dict={}\n",
"#authors('https://pureportal.coventry.ac.uk/en/organisations/faculty-of-engineering-environment-computing/persons/')\n",
"#\n",
"for i in range(0,len(link)):\n",
" url = link[i] \n",
" url = url+\"/publications/\"\n",
" res= requests.get(url)\n",
" soup= BeautifulSoup(res.text,'html.parser')\n",
" aut_name = url.replace('-',' ').split('/')[-3]\n",
" \n",
" for research in soup.find_all('div',{'class': 'result-container'}):\n",
" Name2.append(aut_name)\n",
" Research_title2.append(research.div.h3.a.get_text())\n",
" Link2.append(research.div.h3.a.get('href'))\n",
" \n",
" i+=1\n",
" \n",
"\n",
" \n",
" #return len(Research_title)"
]
},
{
"cell_type": "code",
"execution_count": 32,
"id": "dd4a3424",
"metadata": {},
"outputs": [],
"source": [
"#boss5\n",
"d2={'Author Name':Name2,\n",
" 'Title':Research_title2,\n",
" 'Title Link': Link2}"
]
},
{
"cell_type": "code",
"execution_count": 34,
"id": "ecd66931",
"metadata": {},
"outputs": [],
"source": [
"import json\n",
"def save_data(title,data):\n",
" with open(title,'w',encoding= 'utf-8') as f:\n",
" json.dump(data,f,ensure_ascii=False, indent=2)\n",
" f.close()\n",
" \n"
]
},
{
"cell_type": "code",
"execution_count": 35,
"id": "f183536e",
"metadata": {},
"outputs": [],
"source": [
"#boss6\n",
"save_data('final_file.json',d2)"
]
},
{
"cell_type": "code",
"execution_count": 36,
"id": "8623958e",
"metadata": {},
"outputs": [],
"source": [
"#boss7\n",
"\n",
"f = json.load(open('final_file.json', encoding='utf-8'))\n",
"\n",
"l1,l2,l3 = f['Author Name'] , f['Title Link'] , f['Title']\n",
"\n",
"df2 = pd.DataFrame(list(zip(l1,l2,l3)),columns =['Name', 'TitleLink', 'Title'])"
]
},
{
"cell_type": "code",
"execution_count": 37,
"id": "c54f86af",
"metadata": {},
"outputs": [],
"source": [
"#boss8\n",
"df = pd.DataFrame(list(zip(name, link)),columns =['Name', 'Authlink'])"
]
},
{
"cell_type": "code",
"execution_count": 38,
"id": "1edc7b29",
"metadata": {},
"outputs": [],
"source": [
"#boss9\n",
"df2['Name'] = df2['Name'].str.upper()"
]
},
{
"cell_type": "code",
"execution_count": 39,
"id": "0a2e9c24",
"metadata": {},
"outputs": [],
"source": [
"#boss10\n",
"df['Name'] = df['Name'].str.upper()"
]
},
{
"cell_type": "code",
"execution_count": 40,
"id": "b64f18fe",
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>Name</th>\n",
" <th>Authlink</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>MOHSEN ABBASZADEH</td>\n",
" <td>https://pureportal.coventry.ac.uk/en/persons/m...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>MOHAMED ABDELSHAFY</td>\n",
" <td>https://pureportal.coventry.ac.uk/en/persons/m...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>ESSAM ABO-SERIE</td>\n",
" <td>https://pureportal.coventry.ac.uk/en/persons/e...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>ELIJAH ACQUAH-ANDOH</td>\n",
" <td>https://pureportal.coventry.ac.uk/en/persons/e...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>ZAHIR AHMAD</td>\n",
" <td>https://pureportal.coventry.ac.uk/en/persons/z...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>...</th>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>279</th>\n",
" <td>SEONGKI YOO</td>\n",
" <td>https://pureportal.coventry.ac.uk/en/persons/s...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>280</th>\n",
" <td>TONGYAN ZENG</td>\n",
" <td>https://pureportal.coventry.ac.uk/en/persons/t...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>281</th>\n",
" <td>VINCENT ZHANG</td>\n",
" <td>https://pureportal.coventry.ac.uk/en/persons/v...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>282</th>\n",
" <td>QIAN ZHOU</td>\n",
" <td>https://pureportal.coventry.ac.uk/en/persons/q...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>283</th>\n",
" <td>SHAHEER ZUBAIRI</td>\n",
" <td>https://pureportal.coventry.ac.uk/en/persons/s...</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"<p>284 rows × 2 columns</p>\n",
"</div>"
],
"text/plain": [
" Name Authlink\n",
"0 MOHSEN ABBASZADEH https://pureportal.coventry.ac.uk/en/persons/m...\n",
"1 MOHAMED ABDELSHAFY https://pureportal.coventry.ac.uk/en/persons/m...\n",
"2 ESSAM ABO-SERIE https://pureportal.coventry.ac.uk/en/persons/e...\n",
"3 ELIJAH ACQUAH-ANDOH https://pureportal.coventry.ac.uk/en/persons/e...\n",
"4 ZAHIR AHMAD https://pureportal.coventry.ac.uk/en/persons/z...\n",
".. ... ...\n",
"279 SEONGKI YOO https://pureportal.coventry.ac.uk/en/persons/s...\n",
"280 TONGYAN ZENG https://pureportal.coventry.ac.uk/en/persons/t...\n",
"281 VINCENT ZHANG https://pureportal.coventry.ac.uk/en/persons/v...\n",
"282 QIAN ZHOU https://pureportal.coventry.ac.uk/en/persons/q...\n",
"283 SHAHEER ZUBAIRI https://pureportal.coventry.ac.uk/en/persons/s...\n",
"\n",
"[284 rows x 2 columns]"
]
},
"execution_count": 40,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"df"
]
},
{
"cell_type": "code",
"execution_count": 171,
"id": "40a3ecdc",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"284"
]
},
"execution_count": 171,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"len(df.Name.unique())"
]
},
{
"cell_type": "code",
"execution_count": 43,
"id": "cb464dbe",
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>Name</th>\n",
" <th>TitleLink</th>\n",
" <th>Title</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>MOHSEN ABBASZADEH</td>\n",
" <td>https://pureportal.coventry.ac.uk/en/publicati...</td>\n",
" <td>Enhancing CO2 solubility in the aquifer with t...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>MOHSEN ABBASZADEH</td>\n",
" <td>https://pureportal.coventry.ac.uk/en/publicati...</td>\n",
" <td>Investigating the Impact of Temperature on Rel...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>MOHSEN ABBASZADEH</td>\n",
" <td>https://pureportal.coventry.ac.uk/en/publicati...</td>\n",
" <td>The influence of temperature on wettability al...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>MOHSEN ABBASZADEH</td>\n",
" <td>https://pureportal.coventry.ac.uk/en/publicati...</td>\n",
" <td>Accelerating CO2 Solubility in Brine With Low ...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>MOHSEN ABBASZADEH</td>\n",
" <td>https://pureportal.coventry.ac.uk/en/publicati...</td>\n",
" <td>Investigating The Impact Of Relative Permeabil...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>...</th>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3587</th>\n",
" <td>SHAHEER ZUBAIRI</td>\n",
" <td>https://pureportal.coventry.ac.uk/en/publicati...</td>\n",
" <td>A Sensitivity Approach for Eliminating Clashes...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3588</th>\n",
" <td>SHAHEER ZUBAIRI</td>\n",
" <td>https://pureportal.coventry.ac.uk/en/publicati...</td>\n",
" <td>Using parametric sensitivity analysis to detec...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3589</th>\n",
" <td>SHAHEER ZUBAIRI</td>\n",
" <td>https://pureportal.coventry.ac.uk/en/publicati...</td>\n",
" <td>Design Intent for CAD model assemblies</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3590</th>\n",
" <td>SHAHEER ZUBAIRI</td>\n",
" <td>https://pureportal.coventry.ac.uk/en/publicati...</td>\n",
" <td>Implementing a maths support system for first-...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3591</th>\n",
" <td>SHAHEER ZUBAIRI</td>\n",
" <td>https://pureportal.coventry.ac.uk/en/publicati...</td>\n",
" <td>Sensitivity approach for automatically elimina...</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"<p>3592 rows × 3 columns</p>\n",
"</div>"
],
"text/plain": [
" Name TitleLink \\\n",
"0 MOHSEN ABBASZADEH https://pureportal.coventry.ac.uk/en/publicati... \n",
"1 MOHSEN ABBASZADEH https://pureportal.coventry.ac.uk/en/publicati... \n",
"2 MOHSEN ABBASZADEH https://pureportal.coventry.ac.uk/en/publicati... \n",
"3 MOHSEN ABBASZADEH https://pureportal.coventry.ac.uk/en/publicati... \n",
"4 MOHSEN ABBASZADEH https://pureportal.coventry.ac.uk/en/publicati... \n",
"... ... ... \n",
"3587 SHAHEER ZUBAIRI https://pureportal.coventry.ac.uk/en/publicati... \n",
"3588 SHAHEER ZUBAIRI https://pureportal.coventry.ac.uk/en/publicati... \n",
"3589 SHAHEER ZUBAIRI https://pureportal.coventry.ac.uk/en/publicati... \n",
"3590 SHAHEER ZUBAIRI https://pureportal.coventry.ac.uk/en/publicati... \n",
"3591 SHAHEER ZUBAIRI https://pureportal.coventry.ac.uk/en/publicati... \n",
"\n",
" Title \n",
"0 Enhancing CO2 solubility in the aquifer with t... \n",
"1 Investigating the Impact of Temperature on Rel... \n",
"2 The influence of temperature on wettability al... \n",
"3 Accelerating CO2 Solubility in Brine With Low ... \n",
"4 Investigating The Impact Of Relative Permeabil... \n",
"... ... \n",
"3587 A Sensitivity Approach for Eliminating Clashes... \n",
"3588 Using parametric sensitivity analysis to detec... \n",
"3589 Design Intent for CAD model assemblies \n",
"3590 Implementing a maths support system for first-... \n",
"3591 Sensitivity approach for automatically elimina... \n",
"\n",
"[3592 rows x 3 columns]"
]
},
"execution_count": 43,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"df2"
]
},
{
"cell_type": "code",
"execution_count": 41,
"id": "f4561eed",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"202"
]
},
"execution_count": 41,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"len(df2.Name.unique())"
]
},
{
"cell_type": "code",
"execution_count": 42,
"id": "92c39bae",
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>Name</th>\n",
" <th>Authlink</th>\n",
" <th>TitleLink</th>\n",
" <th>Title</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>MOHSEN ABBASZADEH</td>\n",
" <td>https://pureportal.coventry.ac.uk/en/persons/m...</td>\n",
" <td>https://pureportal.coventry.ac.uk/en/publicati...</td>\n",
" <td>Enhancing CO2 solubility in the aquifer with t...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>MOHSEN ABBASZADEH</td>\n",
" <td>https://pureportal.coventry.ac.uk/en/persons/m...</td>\n",
" <td>https://pureportal.coventry.ac.uk/en/publicati...</td>\n",
" <td>Investigating the Impact of Temperature on Rel...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>MOHSEN ABBASZADEH</td>\n",
" <td>https://pureportal.coventry.ac.uk/en/persons/m...</td>\n",
" <td>https://pureportal.coventry.ac.uk/en/publicati...</td>\n",
" <td>The influence of temperature on wettability al...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>MOHSEN ABBASZADEH</td>\n",
" <td>https://pureportal.coventry.ac.uk/en/persons/m...</td>\n",
" <td>https://pureportal.coventry.ac.uk/en/publicati...</td>\n",
" <td>Accelerating CO2 Solubility in Brine With Low ...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>MOHSEN ABBASZADEH</td>\n",
" <td>https://pureportal.coventry.ac.uk/en/persons/m...</td>\n",
" <td>https://pureportal.coventry.ac.uk/en/publicati...</td>\n",
" <td>Investigating The Impact Of Relative Permeabil...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>...</th>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3091</th>\n",
" <td>SHAHEER ZUBAIRI</td>\n",
" <td>https://pureportal.coventry.ac.uk/en/persons/s...</td>\n",
" <td>https://pureportal.coventry.ac.uk/en/publicati...</td>\n",
" <td>A Sensitivity Approach for Eliminating Clashes...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3092</th>\n",
" <td>SHAHEER ZUBAIRI</td>\n",
" <td>https://pureportal.coventry.ac.uk/en/persons/s...</td>\n",
" <td>https://pureportal.coventry.ac.uk/en/publicati...</td>\n",
" <td>Using parametric sensitivity analysis to detec...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3093</th>\n",
" <td>SHAHEER ZUBAIRI</td>\n",
" <td>https://pureportal.coventry.ac.uk/en/persons/s...</td>\n",
" <td>https://pureportal.coventry.ac.uk/en/publicati...</td>\n",
" <td>Design Intent for CAD model assemblies</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3094</th>\n",
" <td>SHAHEER ZUBAIRI</td>\n",
" <td>https://pureportal.coventry.ac.uk/en/persons/s...</td>\n",
" <td>https://pureportal.coventry.ac.uk/en/publicati...</td>\n",
" <td>Implementing a maths support system for first-...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3095</th>\n",
" <td>SHAHEER ZUBAIRI</td>\n",
" <td>https://pureportal.coventry.ac.uk/en/persons/s...</td>\n",
" <td>https://pureportal.coventry.ac.uk/en/publicati...</td>\n",
" <td>Sensitivity approach for automatically elimina...</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"<p>3096 rows × 4 columns</p>\n",
"</div>"
],
"text/plain": [
" Name Authlink \\\n",
"0 MOHSEN ABBASZADEH https://pureportal.coventry.ac.uk/en/persons/m... \n",
"1 MOHSEN ABBASZADEH https://pureportal.coventry.ac.uk/en/persons/m... \n",
"2 MOHSEN ABBASZADEH https://pureportal.coventry.ac.uk/en/persons/m... \n",
"3 MOHSEN ABBASZADEH https://pureportal.coventry.ac.uk/en/persons/m... \n",
"4 MOHSEN ABBASZADEH https://pureportal.coventry.ac.uk/en/persons/m... \n",
"... ... ... \n",
"3091 SHAHEER ZUBAIRI https://pureportal.coventry.ac.uk/en/persons/s... \n",
"3092 SHAHEER ZUBAIRI https://pureportal.coventry.ac.uk/en/persons/s... \n",
"3093 SHAHEER ZUBAIRI https://pureportal.coventry.ac.uk/en/persons/s... \n",
"3094 SHAHEER ZUBAIRI https://pureportal.coventry.ac.uk/en/persons/s... \n",
"3095 SHAHEER ZUBAIRI https://pureportal.coventry.ac.uk/en/persons/s... \n",
"\n",
" TitleLink \\\n",
"0 https://pureportal.coventry.ac.uk/en/publicati... \n",
"1 https://pureportal.coventry.ac.uk/en/publicati... \n",
"2 https://pureportal.coventry.ac.uk/en/publicati... \n",
"3 https://pureportal.coventry.ac.uk/en/publicati... \n",
"4 https://pureportal.coventry.ac.uk/en/publicati... \n",
"... ... \n",
"3091 https://pureportal.coventry.ac.uk/en/publicati... \n",
"3092 https://pureportal.coventry.ac.uk/en/publicati... \n",
"3093 https://pureportal.coventry.ac.uk/en/publicati... \n",
"3094 https://pureportal.coventry.ac.uk/en/publicati... \n",
"3095 https://pureportal.coventry.ac.uk/en/publicati... \n",
"\n",
" Title \n",
"0 Enhancing CO2 solubility in the aquifer with t... \n",
"1 Investigating the Impact of Temperature on Rel... \n",
"2 The influence of temperature on wettability al... \n",
"3 Accelerating CO2 Solubility in Brine With Low ... \n",
"4 Investigating The Impact Of Relative Permeabil... \n",
"... ... \n",
"3091 A Sensitivity Approach for Eliminating Clashes... \n",
"3092 Using parametric sensitivity analysis to detec... \n",
"3093 Design Intent for CAD model assemblies \n",
"3094 Implementing a maths support system for first-... \n",
"3095 Sensitivity approach for automatically elimina... \n",
"\n",
"[3096 rows x 4 columns]"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"#boss11\n",
"frames = [df, df2]\n",
" \n",
"result = pd.merge(df, df2 ,how = 'inner' , on = 'Name')\n",
"display(result)"
]
},
{
"cell_type": "code",
"execution_count": 44,
"id": "2c2f169f",
"metadata": {},
"outputs": [],
"source": [
"#boss12\n",
"result.to_csv('full_file3.txt' , sep=\",\")"
]
},
{
"cell_type": "code",
"execution_count": 12,
"id": "d42c9a0e",
"metadata": {},
"outputs": [
{
"ename": "NameError",
"evalue": "name 'df' is not defined",
"output_type": "error",
"traceback": [
"\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[1;31mNameError\u001b[0m Traceback (most recent call last)",
"\u001b[1;32m<ipython-input-12-1bc952dbe894>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m 2\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mstring\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 3\u001b[0m \u001b[0mnorm_docs\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 4\u001b[1;33m \u001b[1;32mfor\u001b[0m \u001b[0md\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mdf\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'Research Title'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 5\u001b[0m \u001b[1;31m#Unicode removal\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 6\u001b[0m \u001b[0mdocument_test\u001b[0m\u001b[1;33m=\u001b[0m \u001b[0mre\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msub\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34mr'[^\\x00-\\x7F]+'\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;34m''\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0md\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
"\u001b[1;31mNameError\u001b[0m: name 'df' is not defined"
]
}
],
"source": [
"from nltk.corpus import stopwords\n",
"import string\n",
"norm_docs = []\n",
"for d in df['Research Title']:\n",
" #Unicode removal\n",
" document_test= re.sub(r'[^\\x00-\\x7F]+','', d)\n",
" #Lower case\n",
" document_test=document_test.lower()\n",
" #Remove punctuations\n",
" document_test=re.sub(r'[%s]'% re.escape(string.punctuation),'',document_test)\n",
" #Remove stop words\n",
" pattern = re.compile(r'\\b(' + r'|'.join(stopwords.words('english')) + r')b\\s*')\n",
" txt = pattern.sub('',document_test)\n",
" norm_docs.append(txt)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "df44a279",
"metadata": {},
"outputs": [],
"source": [
"#Data Cleaning\n",
"\n",
"from nltk.tokenize import sent_tokenize, word_tokenize, TweetTokenizer\n",
"from nltk.stem.snowball import SnowballStemmer\n",
"stem = SnowballStemmer('english')\n",
"\n",
"stem_sentence = []\n",
"Z = []\n",
"for i in sentence:\n",
" s=[]\n",
" tokens= word_tokenize(i)\n",
" tokens\n",
" for word in tokens:\n",
" stem_sentence.append(stemmer.stem(word))\n",
" s=' '.join(stem_sentence)\n",
" Z.insert(count,s) \n",
" count+=1\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "231e5563",
"metadata": {},
"outputs": [],
"source": [
"vectorizer = Tfidvectorizer\n",
"vectors = vectorizer.fit_transform(norm_docs)\n",
"features = vectorizer.get_feature_names()\n",
"dense = vectors.todense()\n",
"denselist = dense.tolist()\n",
"tfidf = pd.DataFrame(denselist,columns-feature_names)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "37b54e64",
"metadata": {},
"outputs": [],
"source": [
"tfidf.info()"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "ed6ca828",
"metadata": {},
"outputs": [],
"source": [
"tfidf.head()"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "11decc10",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "bb59d33c",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "e3347720",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "935b4443",
"metadata": {},
"outputs": [],
"source": [
"from bs4 import BeautifulSoup as bs\n",
"import requests\n",
"from collections import defaultdict\n",
"import warnings\n",
"import xml\n",
"import pandas as pd\n",
"import logging\n",
"\n",
"\n",
"\n",
"urls = [\n",
" \"https://scholar.google.com/citations?view_op=view_org&hl=en&org=9117984065169182779&before_author=RmH6_ysYAAAJ&astart=0\",\n",
" \"https://scholar.google.com/citations?view_op=view_org&hl=en&org=9117984065169182779&after_author=8fsEAMjl__8J&astart=10\",\n",
" \"https://scholar.google.com/citations?view_op=view_org&hl=en&org=9117984065169182779&after_author=ZKcRAAbu__8J&astart=20\",\n",
" \"https://scholar.google.com/citations?view_op=view_org&hl=en&org=9117984065169182779&after_author=oHpYAInx__8J&astart=30\",\n",
" \"https://scholar.google.com/citations?view_op=view_org&hl=en&org=9117984065169182779&after_author=7QcEAOrz__8J&astart=40\"\n",
"]\n",
"names =[]\n",
"cited_by = []\n",
"publications = []\n",
"out = \"output.dat\"\n",
"file = open(out, 'w')\n",
"\n",
"\n",
"for url in urls:\n",
" response = requests.get(url)\n",
" html = response.content\n",
" soup = bs(html, \"html.parser\")\n",
"\n",
" file.write(\"<parseRoot>\") \n",
" file.write(\"<page>\\n\")\n",
" for div_names in soup.find_all(\"a\",class_=\"contributionTool\"):\n",
" \n",
" name = div_names.get_text(strip=True)\n",
" file.write(\"<lecnames>{}<lecnames>\\n\".format(name))\n",
"\n",
" names.append(div_names.get_text(strip=True))\n",
"\n",
" for div_cited in soup.find_all(\"div\",class_=\"gs_ai_cby\"):\n",
" \n",
" cite = div_cited.get_text(strip=True)\n",
" file.write(\"<id>{}<\\id>\\n\".format(cite))\n",
"\n",
" cited_by.append(cite)\n",
"\n",
" for div_titles in soup.find_all(\"div\",class_=\"gs_ai_int\"):\n",
" one_title =[]\n",
" for link in div_titles.find_all(\"a\",class_=\"gs_ai_one_int\"):\n",
" tile = link.get_text(strip=True)\n",
" file.write(\"<publications>{}<publications>\\n\".format(tile))\n",
" one_title.append(link.get_text(strip=True))\n",
" publications.append(one_title)\n",
" file.write(\"</parseRoot>\") \n",
" file.close()\n",
"\n",
"pd_names = pd.Series(names)\n",
"pd_publications = pd.Series(publications)\n",
"pd_citations = pd.Series(cited_by)\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "75da2998",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "cd5d9b78",
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.8.8"
}
},
"nbformat": 4,
"nbformat_minor": 5
}