mirror of
https://github.com/Bunsly/JobSpy.git
synced 2026-03-05 03:54:31 -08:00
Compare commits
13 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
59f739018a | ||
|
a37e7f235e | ||
|
|
690739e858 | ||
|
43eb2fe0e8 | ||
|
|
e50227bba6 | ||
|
|
45c2d76e15 | ||
|
|
fd883178be | ||
|
|
70e2218c67 | ||
|
|
d6947ecdd7 | ||
|
|
5191658562 | ||
|
|
1c264b8c58 | ||
|
|
1598d4ff63 | ||
|
|
bf2460684b |
@@ -2,18 +2,19 @@
|
|||||||
"cells": [
|
"cells": [
|
||||||
{
|
{
|
||||||
"cell_type": "code",
|
"cell_type": "code",
|
||||||
"execution_count": 1,
|
"execution_count": null,
|
||||||
"id": "00a94b47-f47b-420f-ba7e-714ef219c006",
|
"id": "00a94b47-f47b-420f-ba7e-714ef219c006",
|
||||||
"metadata": {},
|
"metadata": {},
|
||||||
"outputs": [],
|
"outputs": [],
|
||||||
"source": [
|
"source": [
|
||||||
"from jobspy import scrape_jobs\n",
|
"from jobspy import scrape_jobs\n",
|
||||||
"import pandas as pd"
|
"import pandas as pd\n",
|
||||||
|
"from IPython.display import display, HTML\n"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"cell_type": "code",
|
"cell_type": "code",
|
||||||
"execution_count": 2,
|
"execution_count": null,
|
||||||
"id": "9f773e6c-d9fc-42cc-b0ef-63b739e78435",
|
"id": "9f773e6c-d9fc-42cc-b0ef-63b739e78435",
|
||||||
"metadata": {},
|
"metadata": {},
|
||||||
"outputs": [],
|
"outputs": [],
|
||||||
@@ -26,643 +27,126 @@
|
|||||||
},
|
},
|
||||||
{
|
{
|
||||||
"cell_type": "code",
|
"cell_type": "code",
|
||||||
"execution_count": 5,
|
"execution_count": null,
|
||||||
"id": "1253c1f8-9437-492e-9dd3-e7fe51099420",
|
"id": "1253c1f8-9437-492e-9dd3-e7fe51099420",
|
||||||
"metadata": {},
|
"metadata": {},
|
||||||
"outputs": [
|
"outputs": [],
|
||||||
{
|
|
||||||
"data": {
|
|
||||||
"text/html": [
|
|
||||||
"<div>\n",
|
|
||||||
"<style scoped>\n",
|
|
||||||
" .dataframe tbody tr th:only-of-type {\n",
|
|
||||||
" vertical-align: middle;\n",
|
|
||||||
" }\n",
|
|
||||||
"\n",
|
|
||||||
" .dataframe tbody tr th {\n",
|
|
||||||
" vertical-align: top;\n",
|
|
||||||
" }\n",
|
|
||||||
"\n",
|
|
||||||
" .dataframe thead th {\n",
|
|
||||||
" text-align: right;\n",
|
|
||||||
" }\n",
|
|
||||||
"</style>\n",
|
|
||||||
"<table border=\"1\" class=\"dataframe\">\n",
|
|
||||||
" <thead>\n",
|
|
||||||
" <tr style=\"text-align: right;\">\n",
|
|
||||||
" <th></th>\n",
|
|
||||||
" <th>site</th>\n",
|
|
||||||
" <th>title</th>\n",
|
|
||||||
" <th>company_name</th>\n",
|
|
||||||
" <th>city</th>\n",
|
|
||||||
" <th>state</th>\n",
|
|
||||||
" <th>job_type</th>\n",
|
|
||||||
" <th>interval</th>\n",
|
|
||||||
" <th>min_amount</th>\n",
|
|
||||||
" <th>max_amount</th>\n",
|
|
||||||
" <th>job_url</th>\n",
|
|
||||||
" <th>description</th>\n",
|
|
||||||
" </tr>\n",
|
|
||||||
" </thead>\n",
|
|
||||||
" <tbody>\n",
|
|
||||||
" <tr>\n",
|
|
||||||
" <th>0</th>\n",
|
|
||||||
" <td>indeed</td>\n",
|
|
||||||
" <td>Mental Health Therapist</td>\n",
|
|
||||||
" <td>Sandstone Care</td>\n",
|
|
||||||
" <td>Broomfield</td>\n",
|
|
||||||
" <td>CO</td>\n",
|
|
||||||
" <td>fulltime</td>\n",
|
|
||||||
" <td>yearly</td>\n",
|
|
||||||
" <td>68000</td>\n",
|
|
||||||
" <td>57500</td>\n",
|
|
||||||
" <td>https://www.indeed.com/viewjob?jk=f5f33d72e030...</td>\n",
|
|
||||||
" <td>Mental Health Therapist- Broomfield, CO Locati...</td>\n",
|
|
||||||
" </tr>\n",
|
|
||||||
" <tr>\n",
|
|
||||||
" <th>1</th>\n",
|
|
||||||
" <td>indeed</td>\n",
|
|
||||||
" <td>.NET Developer</td>\n",
|
|
||||||
" <td>Noir Consulting</td>\n",
|
|
||||||
" <td>Irving</td>\n",
|
|
||||||
" <td>TX</td>\n",
|
|
||||||
" <td>None</td>\n",
|
|
||||||
" <td>yearly</td>\n",
|
|
||||||
" <td>200000</td>\n",
|
|
||||||
" <td>200000</td>\n",
|
|
||||||
" <td>https://www.indeed.com/viewjob?jk=1b22ba65296c...</td>\n",
|
|
||||||
" <td>.NET Software Engineer, C#, WPF - Irving (Tech...</td>\n",
|
|
||||||
" </tr>\n",
|
|
||||||
" <tr>\n",
|
|
||||||
" <th>2</th>\n",
|
|
||||||
" <td>indeed</td>\n",
|
|
||||||
" <td>Senior Software Engineer</td>\n",
|
|
||||||
" <td>Johns Hopkins Applied Physics Laboratory (APL)</td>\n",
|
|
||||||
" <td>Laurel</td>\n",
|
|
||||||
" <td>MD</td>\n",
|
|
||||||
" <td>None</td>\n",
|
|
||||||
" <td>None</td>\n",
|
|
||||||
" <td>None</td>\n",
|
|
||||||
" <td>None</td>\n",
|
|
||||||
" <td>https://www.indeed.com/viewjob?jk=309eed270a88...</td>\n",
|
|
||||||
" <td>Description Are you a communications systems d...</td>\n",
|
|
||||||
" </tr>\n",
|
|
||||||
" <tr>\n",
|
|
||||||
" <th>3</th>\n",
|
|
||||||
" <td>indeed</td>\n",
|
|
||||||
" <td>Front End Developer</td>\n",
|
|
||||||
" <td>Verkada</td>\n",
|
|
||||||
" <td>San Mateo</td>\n",
|
|
||||||
" <td>CA</td>\n",
|
|
||||||
" <td>fulltime</td>\n",
|
|
||||||
" <td>yearly</td>\n",
|
|
||||||
" <td>285000</td>\n",
|
|
||||||
" <td>120000</td>\n",
|
|
||||||
" <td>https://www.indeed.com/viewjob?jk=a3ea45daca75...</td>\n",
|
|
||||||
" <td>Who We Are Verkada is the largest cloud-based ...</td>\n",
|
|
||||||
" </tr>\n",
|
|
||||||
" <tr>\n",
|
|
||||||
" <th>4</th>\n",
|
|
||||||
" <td>indeed</td>\n",
|
|
||||||
" <td>Software Engineer</td>\n",
|
|
||||||
" <td>Adobe</td>\n",
|
|
||||||
" <td>San Jose</td>\n",
|
|
||||||
" <td>CA</td>\n",
|
|
||||||
" <td>fulltime</td>\n",
|
|
||||||
" <td>yearly</td>\n",
|
|
||||||
" <td>142700</td>\n",
|
|
||||||
" <td>73200</td>\n",
|
|
||||||
" <td>https://www.indeed.com/viewjob?jk=0f2dc9901fc7...</td>\n",
|
|
||||||
" <td>Our Company Changing the world through digital...</td>\n",
|
|
||||||
" </tr>\n",
|
|
||||||
" <tr>\n",
|
|
||||||
" <th>5</th>\n",
|
|
||||||
" <td>indeed</td>\n",
|
|
||||||
" <td>Full Stack Developer</td>\n",
|
|
||||||
" <td>Comcast</td>\n",
|
|
||||||
" <td>Philadelphia</td>\n",
|
|
||||||
" <td>PA</td>\n",
|
|
||||||
" <td>fulltime</td>\n",
|
|
||||||
" <td>yearly</td>\n",
|
|
||||||
" <td>184663</td>\n",
|
|
||||||
" <td>78789</td>\n",
|
|
||||||
" <td>https://www.indeed.com/viewjob?jk=eb5c927221eb...</td>\n",
|
|
||||||
" <td>Make your mark at Comcast - a Fortune 30 globa...</td>\n",
|
|
||||||
" </tr>\n",
|
|
||||||
" <tr>\n",
|
|
||||||
" <th>6</th>\n",
|
|
||||||
" <td>indeed</td>\n",
|
|
||||||
" <td>Senior Software Engineer</td>\n",
|
|
||||||
" <td>Smart City Solutions</td>\n",
|
|
||||||
" <td></td>\n",
|
|
||||||
" <td>FL</td>\n",
|
|
||||||
" <td>fulltime</td>\n",
|
|
||||||
" <td>yearly</td>\n",
|
|
||||||
" <td>100000</td>\n",
|
|
||||||
" <td>85000</td>\n",
|
|
||||||
" <td>https://www.indeed.com/viewjob?jk=ba1945f143a1...</td>\n",
|
|
||||||
" <td>Smart City hiring a full stack software develo...</td>\n",
|
|
||||||
" </tr>\n",
|
|
||||||
" <tr>\n",
|
|
||||||
" <th>7</th>\n",
|
|
||||||
" <td>indeed</td>\n",
|
|
||||||
" <td>Computer Engineer</td>\n",
|
|
||||||
" <td>Honeywell</td>\n",
|
|
||||||
" <td></td>\n",
|
|
||||||
" <td>None</td>\n",
|
|
||||||
" <td>fulltime</td>\n",
|
|
||||||
" <td>None</td>\n",
|
|
||||||
" <td>None</td>\n",
|
|
||||||
" <td>None</td>\n",
|
|
||||||
" <td>https://www.indeed.com/viewjob?jk=5a1da623ee75...</td>\n",
|
|
||||||
" <td>Join a team recognized for leadership, innovat...</td>\n",
|
|
||||||
" </tr>\n",
|
|
||||||
" <tr>\n",
|
|
||||||
" <th>8</th>\n",
|
|
||||||
" <td>indeed</td>\n",
|
|
||||||
" <td>Software Engineer</td>\n",
|
|
||||||
" <td>Fidelity Investments</td>\n",
|
|
||||||
" <td>Westlake</td>\n",
|
|
||||||
" <td>TX</td>\n",
|
|
||||||
" <td>None</td>\n",
|
|
||||||
" <td>None</td>\n",
|
|
||||||
" <td>None</td>\n",
|
|
||||||
" <td>None</td>\n",
|
|
||||||
" <td>https://www.indeed.com/viewjob?jk=b600392166bb...</td>\n",
|
|
||||||
" <td>Job Description: Software Engineer in Test The...</td>\n",
|
|
||||||
" </tr>\n",
|
|
||||||
" <tr>\n",
|
|
||||||
" <th>9</th>\n",
|
|
||||||
" <td>indeed</td>\n",
|
|
||||||
" <td>Fpga Engineer</td>\n",
|
|
||||||
" <td>R-DEX Systems, Inc.</td>\n",
|
|
||||||
" <td>Atlanta</td>\n",
|
|
||||||
" <td>GA</td>\n",
|
|
||||||
" <td>fulltime</td>\n",
|
|
||||||
" <td>yearly</td>\n",
|
|
||||||
" <td>160000</td>\n",
|
|
||||||
" <td>120000</td>\n",
|
|
||||||
" <td>https://www.indeed.com/viewjob?jk=a7e9d356c333...</td>\n",
|
|
||||||
" <td>Title: Senior DSP/FPGA Firmware Engineer Descr...</td>\n",
|
|
||||||
" </tr>\n",
|
|
||||||
" <tr>\n",
|
|
||||||
" <th>10</th>\n",
|
|
||||||
" <td>linkedin</td>\n",
|
|
||||||
" <td>Software Engineer</td>\n",
|
|
||||||
" <td>Fieldguide</td>\n",
|
|
||||||
" <td>San Francisco</td>\n",
|
|
||||||
" <td>CA</td>\n",
|
|
||||||
" <td>fulltime</td>\n",
|
|
||||||
" <td>yearly</td>\n",
|
|
||||||
" <td>None</td>\n",
|
|
||||||
" <td>None</td>\n",
|
|
||||||
" <td>https://www.linkedin.com/jobs/view/3696158160</td>\n",
|
|
||||||
" <td>About us:Fieldguide is establishing a new stat...</td>\n",
|
|
||||||
" </tr>\n",
|
|
||||||
" <tr>\n",
|
|
||||||
" <th>11</th>\n",
|
|
||||||
" <td>linkedin</td>\n",
|
|
||||||
" <td>Software Engineer - Early Career</td>\n",
|
|
||||||
" <td>Lockheed Martin</td>\n",
|
|
||||||
" <td>Sunnyvale</td>\n",
|
|
||||||
" <td>CA</td>\n",
|
|
||||||
" <td>fulltime</td>\n",
|
|
||||||
" <td>yearly</td>\n",
|
|
||||||
" <td>None</td>\n",
|
|
||||||
" <td>None</td>\n",
|
|
||||||
" <td>https://www.linkedin.com/jobs/view/3693012711</td>\n",
|
|
||||||
" <td>Description:By bringing together people that u...</td>\n",
|
|
||||||
" </tr>\n",
|
|
||||||
" <tr>\n",
|
|
||||||
" <th>12</th>\n",
|
|
||||||
" <td>linkedin</td>\n",
|
|
||||||
" <td>Software Engineer - Early Career</td>\n",
|
|
||||||
" <td>Lockheed Martin</td>\n",
|
|
||||||
" <td>Edwards</td>\n",
|
|
||||||
" <td>CA</td>\n",
|
|
||||||
" <td>fulltime</td>\n",
|
|
||||||
" <td>yearly</td>\n",
|
|
||||||
" <td>None</td>\n",
|
|
||||||
" <td>None</td>\n",
|
|
||||||
" <td>https://www.linkedin.com/jobs/view/3700669785</td>\n",
|
|
||||||
" <td>Description:By bringing together people that u...</td>\n",
|
|
||||||
" </tr>\n",
|
|
||||||
" <tr>\n",
|
|
||||||
" <th>13</th>\n",
|
|
||||||
" <td>linkedin</td>\n",
|
|
||||||
" <td>Software Engineer - Early Career</td>\n",
|
|
||||||
" <td>Lockheed Martin</td>\n",
|
|
||||||
" <td>Fort Worth</td>\n",
|
|
||||||
" <td>TX</td>\n",
|
|
||||||
" <td>fulltime</td>\n",
|
|
||||||
" <td>yearly</td>\n",
|
|
||||||
" <td>None</td>\n",
|
|
||||||
" <td>None</td>\n",
|
|
||||||
" <td>https://www.linkedin.com/jobs/view/3701770659</td>\n",
|
|
||||||
" <td>Description:By bringing together people that u...</td>\n",
|
|
||||||
" </tr>\n",
|
|
||||||
" <tr>\n",
|
|
||||||
" <th>14</th>\n",
|
|
||||||
" <td>linkedin</td>\n",
|
|
||||||
" <td>Software Engineer - Early Career</td>\n",
|
|
||||||
" <td>Lockheed Martin</td>\n",
|
|
||||||
" <td>Fort Worth</td>\n",
|
|
||||||
" <td>TX</td>\n",
|
|
||||||
" <td>fulltime</td>\n",
|
|
||||||
" <td>yearly</td>\n",
|
|
||||||
" <td>None</td>\n",
|
|
||||||
" <td>None</td>\n",
|
|
||||||
" <td>https://www.linkedin.com/jobs/view/3701769637</td>\n",
|
|
||||||
" <td>Description:By bringing together people that u...</td>\n",
|
|
||||||
" </tr>\n",
|
|
||||||
" <tr>\n",
|
|
||||||
" <th>15</th>\n",
|
|
||||||
" <td>linkedin</td>\n",
|
|
||||||
" <td>Software Engineer - Early Career</td>\n",
|
|
||||||
" <td>Lockheed Martin</td>\n",
|
|
||||||
" <td>Fort Worth</td>\n",
|
|
||||||
" <td>TX</td>\n",
|
|
||||||
" <td>fulltime</td>\n",
|
|
||||||
" <td>yearly</td>\n",
|
|
||||||
" <td>None</td>\n",
|
|
||||||
" <td>None</td>\n",
|
|
||||||
" <td>https://www.linkedin.com/jobs/view/3701772329</td>\n",
|
|
||||||
" <td>Description:By bringing together people that u...</td>\n",
|
|
||||||
" </tr>\n",
|
|
||||||
" <tr>\n",
|
|
||||||
" <th>16</th>\n",
|
|
||||||
" <td>linkedin</td>\n",
|
|
||||||
" <td>Software Engineer - Early Career</td>\n",
|
|
||||||
" <td>Lockheed Martin</td>\n",
|
|
||||||
" <td>Fort Worth</td>\n",
|
|
||||||
" <td>TX</td>\n",
|
|
||||||
" <td>fulltime</td>\n",
|
|
||||||
" <td>yearly</td>\n",
|
|
||||||
" <td>None</td>\n",
|
|
||||||
" <td>None</td>\n",
|
|
||||||
" <td>https://www.linkedin.com/jobs/view/3701775201</td>\n",
|
|
||||||
" <td>Description:By bringing together people that u...</td>\n",
|
|
||||||
" </tr>\n",
|
|
||||||
" <tr>\n",
|
|
||||||
" <th>17</th>\n",
|
|
||||||
" <td>linkedin</td>\n",
|
|
||||||
" <td>Software Engineer</td>\n",
|
|
||||||
" <td>SpiderOak</td>\n",
|
|
||||||
" <td>Austin</td>\n",
|
|
||||||
" <td>TX</td>\n",
|
|
||||||
" <td>fulltime</td>\n",
|
|
||||||
" <td>yearly</td>\n",
|
|
||||||
" <td>None</td>\n",
|
|
||||||
" <td>None</td>\n",
|
|
||||||
" <td>https://www.linkedin.com/jobs/view/3707174719</td>\n",
|
|
||||||
" <td>We're only as strong as our weakest link.In th...</td>\n",
|
|
||||||
" </tr>\n",
|
|
||||||
" <tr>\n",
|
|
||||||
" <th>18</th>\n",
|
|
||||||
" <td>linkedin</td>\n",
|
|
||||||
" <td>Full-Stack Software Engineer</td>\n",
|
|
||||||
" <td>Rain</td>\n",
|
|
||||||
" <td>New York</td>\n",
|
|
||||||
" <td>NY</td>\n",
|
|
||||||
" <td>fulltime</td>\n",
|
|
||||||
" <td>yearly</td>\n",
|
|
||||||
" <td>None</td>\n",
|
|
||||||
" <td>None</td>\n",
|
|
||||||
" <td>https://www.linkedin.com/jobs/view/3696158877</td>\n",
|
|
||||||
" <td>Rain’s mission is to create the fastest and ea...</td>\n",
|
|
||||||
" </tr>\n",
|
|
||||||
" <tr>\n",
|
|
||||||
" <th>19</th>\n",
|
|
||||||
" <td>linkedin</td>\n",
|
|
||||||
" <td>Software Engineer</td>\n",
|
|
||||||
" <td>Nike</td>\n",
|
|
||||||
" <td>Portland</td>\n",
|
|
||||||
" <td>OR</td>\n",
|
|
||||||
" <td>contract</td>\n",
|
|
||||||
" <td>yearly</td>\n",
|
|
||||||
" <td>None</td>\n",
|
|
||||||
" <td>None</td>\n",
|
|
||||||
" <td>https://www.linkedin.com/jobs/view/3693340247</td>\n",
|
|
||||||
" <td>Work options: FlexibleWe consider remote, on-p...</td>\n",
|
|
||||||
" </tr>\n",
|
|
||||||
" <tr>\n",
|
|
||||||
" <th>20</th>\n",
|
|
||||||
" <td>zip_recruiter</td>\n",
|
|
||||||
" <td>Software Engineer - New Grad</td>\n",
|
|
||||||
" <td>ZipRecruiter</td>\n",
|
|
||||||
" <td>Santa Monica</td>\n",
|
|
||||||
" <td>CA</td>\n",
|
|
||||||
" <td>fulltime</td>\n",
|
|
||||||
" <td>yearly</td>\n",
|
|
||||||
" <td>130000</td>\n",
|
|
||||||
" <td>150000</td>\n",
|
|
||||||
" <td>https://www.ziprecruiter.com/c/ZipRecruiter/Jo...</td>\n",
|
|
||||||
" <td>Demonstrated foundation in software engineerin...</td>\n",
|
|
||||||
" </tr>\n",
|
|
||||||
" <tr>\n",
|
|
||||||
" <th>21</th>\n",
|
|
||||||
" <td>zip_recruiter</td>\n",
|
|
||||||
" <td>Full Stack Software Engineer</td>\n",
|
|
||||||
" <td>ZipRecruiter</td>\n",
|
|
||||||
" <td>Phoenix</td>\n",
|
|
||||||
" <td>AZ</td>\n",
|
|
||||||
" <td>fulltime</td>\n",
|
|
||||||
" <td>yearly</td>\n",
|
|
||||||
" <td>105000</td>\n",
|
|
||||||
" <td>145000</td>\n",
|
|
||||||
" <td>https://www.ziprecruiter.com/c/ZipRecruiter/Jo...</td>\n",
|
|
||||||
" <td>Experience in client side development using Re...</td>\n",
|
|
||||||
" </tr>\n",
|
|
||||||
" <tr>\n",
|
|
||||||
" <th>22</th>\n",
|
|
||||||
" <td>zip_recruiter</td>\n",
|
|
||||||
" <td>Software Developer | Onsite | Omaha, NE - Omaha</td>\n",
|
|
||||||
" <td>OneStaff Medical</td>\n",
|
|
||||||
" <td>Omaha</td>\n",
|
|
||||||
" <td>NE</td>\n",
|
|
||||||
" <td>fulltime</td>\n",
|
|
||||||
" <td>yearly</td>\n",
|
|
||||||
" <td>60000</td>\n",
|
|
||||||
" <td>110000</td>\n",
|
|
||||||
" <td>https://www.ziprecruiter.com/c/OneStaff-Medica...</td>\n",
|
|
||||||
" <td>We are looking for a well-rounded Software Dev...</td>\n",
|
|
||||||
" </tr>\n",
|
|
||||||
" <tr>\n",
|
|
||||||
" <th>23</th>\n",
|
|
||||||
" <td>zip_recruiter</td>\n",
|
|
||||||
" <td>Senior Software Engineer, Onsite [Real-time]</td>\n",
|
|
||||||
" <td>Raytheon</td>\n",
|
|
||||||
" <td>McKinney</td>\n",
|
|
||||||
" <td>TX</td>\n",
|
|
||||||
" <td>fulltime</td>\n",
|
|
||||||
" <td>yearly</td>\n",
|
|
||||||
" <td>116000</td>\n",
|
|
||||||
" <td>153000</td>\n",
|
|
||||||
" <td>https://jsv3.recruitics.com/redirect?rx_cid=34...</td>\n",
|
|
||||||
" <td>By joining the Silent Knight team as a Senior ...</td>\n",
|
|
||||||
" </tr>\n",
|
|
||||||
" <tr>\n",
|
|
||||||
" <th>24</th>\n",
|
|
||||||
" <td>zip_recruiter</td>\n",
|
|
||||||
" <td>Senior Software Engineer - TS/SCI **Minimum $2...</td>\n",
|
|
||||||
" <td>Raytheon</td>\n",
|
|
||||||
" <td>Dallas</td>\n",
|
|
||||||
" <td>TX</td>\n",
|
|
||||||
" <td>fulltime</td>\n",
|
|
||||||
" <td>yearly</td>\n",
|
|
||||||
" <td>122000</td>\n",
|
|
||||||
" <td>162000</td>\n",
|
|
||||||
" <td>https://jsv3.recruitics.com/redirect?rx_cid=34...</td>\n",
|
|
||||||
" <td>Object Oriented Programming using C++ with Lin...</td>\n",
|
|
||||||
" </tr>\n",
|
|
||||||
" <tr>\n",
|
|
||||||
" <th>25</th>\n",
|
|
||||||
" <td>zip_recruiter</td>\n",
|
|
||||||
" <td>Software Engineer III (full stack, AI/ML, Djan...</td>\n",
|
|
||||||
" <td>Ayahealthcare</td>\n",
|
|
||||||
" <td>Remote</td>\n",
|
|
||||||
" <td>OR</td>\n",
|
|
||||||
" <td>None</td>\n",
|
|
||||||
" <td>yearly</td>\n",
|
|
||||||
" <td>156000</td>\n",
|
|
||||||
" <td>165000</td>\n",
|
|
||||||
" <td>https://click.appcast.io/track/hcbh0qq?cs=ngp&...</td>\n",
|
|
||||||
" <td>The Software Engineer III will be an integral ...</td>\n",
|
|
||||||
" </tr>\n",
|
|
||||||
" <tr>\n",
|
|
||||||
" <th>26</th>\n",
|
|
||||||
" <td>zip_recruiter</td>\n",
|
|
||||||
" <td>Software Engineer Full Stack</td>\n",
|
|
||||||
" <td>Generac Power Systems</td>\n",
|
|
||||||
" <td>Denver</td>\n",
|
|
||||||
" <td>CO</td>\n",
|
|
||||||
" <td>fulltime</td>\n",
|
|
||||||
" <td>yearly</td>\n",
|
|
||||||
" <td>90000</td>\n",
|
|
||||||
" <td>115000</td>\n",
|
|
||||||
" <td>https://www.ziprecruiter.com/c/Generac-Power-S...</td>\n",
|
|
||||||
" <td>As a Software Engineer on the Energy Technolog...</td>\n",
|
|
||||||
" </tr>\n",
|
|
||||||
" <tr>\n",
|
|
||||||
" <th>27</th>\n",
|
|
||||||
" <td>zip_recruiter</td>\n",
|
|
||||||
" <td>Embedded Software Engineer (Fort Worth, TX or ...</td>\n",
|
|
||||||
" <td>Kubota</td>\n",
|
|
||||||
" <td>Fort Worth</td>\n",
|
|
||||||
" <td>TX</td>\n",
|
|
||||||
" <td>fulltime</td>\n",
|
|
||||||
" <td>yearly</td>\n",
|
|
||||||
" <td>122000</td>\n",
|
|
||||||
" <td>167000</td>\n",
|
|
||||||
" <td>https://us62e2.dayforcehcm.com/CandidatePortal...</td>\n",
|
|
||||||
" <td>Work with a cross-functional team to design, t...</td>\n",
|
|
||||||
" </tr>\n",
|
|
||||||
" <tr>\n",
|
|
||||||
" <th>28</th>\n",
|
|
||||||
" <td>zip_recruiter</td>\n",
|
|
||||||
" <td>Senior Software Engineer (FT)</td>\n",
|
|
||||||
" <td>National Indoor RV Center</td>\n",
|
|
||||||
" <td>Lewisville</td>\n",
|
|
||||||
" <td>TX</td>\n",
|
|
||||||
" <td>fulltime</td>\n",
|
|
||||||
" <td>yearly</td>\n",
|
|
||||||
" <td>125000</td>\n",
|
|
||||||
" <td>0</td>\n",
|
|
||||||
" <td>https://www.ziprecruiter.com/c/National-Indoor...</td>\n",
|
|
||||||
" <td>As a Senior Software Engineer, you will: * Des...</td>\n",
|
|
||||||
" </tr>\n",
|
|
||||||
" <tr>\n",
|
|
||||||
" <th>29</th>\n",
|
|
||||||
" <td>zip_recruiter</td>\n",
|
|
||||||
" <td>2024 Next Gen IT Program | Software Engineerin...</td>\n",
|
|
||||||
" <td>Southern Glazer's Wine & Spirits</td>\n",
|
|
||||||
" <td>Dallas</td>\n",
|
|
||||||
" <td>TX</td>\n",
|
|
||||||
" <td>None</td>\n",
|
|
||||||
" <td>yearly</td>\n",
|
|
||||||
" <td>70000</td>\n",
|
|
||||||
" <td>0</td>\n",
|
|
||||||
" <td>https://click.appcast.io/track/hdsbnae?cs=b4&j...</td>\n",
|
|
||||||
" <td>Finally, through the work assigned, the analys...</td>\n",
|
|
||||||
" </tr>\n",
|
|
||||||
" </tbody>\n",
|
|
||||||
"</table>\n",
|
|
||||||
"</div>"
|
|
||||||
],
|
|
||||||
"text/plain": [
|
|
||||||
" site title \\\n",
|
|
||||||
"0 indeed Mental Health Therapist \n",
|
|
||||||
"1 indeed .NET Developer \n",
|
|
||||||
"2 indeed Senior Software Engineer \n",
|
|
||||||
"3 indeed Front End Developer \n",
|
|
||||||
"4 indeed Software Engineer \n",
|
|
||||||
"5 indeed Full Stack Developer \n",
|
|
||||||
"6 indeed Senior Software Engineer \n",
|
|
||||||
"7 indeed Computer Engineer \n",
|
|
||||||
"8 indeed Software Engineer \n",
|
|
||||||
"9 indeed Fpga Engineer \n",
|
|
||||||
"10 linkedin Software Engineer \n",
|
|
||||||
"11 linkedin Software Engineer - Early Career \n",
|
|
||||||
"12 linkedin Software Engineer - Early Career \n",
|
|
||||||
"13 linkedin Software Engineer - Early Career \n",
|
|
||||||
"14 linkedin Software Engineer - Early Career \n",
|
|
||||||
"15 linkedin Software Engineer - Early Career \n",
|
|
||||||
"16 linkedin Software Engineer - Early Career \n",
|
|
||||||
"17 linkedin Software Engineer \n",
|
|
||||||
"18 linkedin Full-Stack Software Engineer \n",
|
|
||||||
"19 linkedin Software Engineer \n",
|
|
||||||
"20 zip_recruiter Software Engineer - New Grad \n",
|
|
||||||
"21 zip_recruiter Full Stack Software Engineer \n",
|
|
||||||
"22 zip_recruiter Software Developer | Onsite | Omaha, NE - Omaha \n",
|
|
||||||
"23 zip_recruiter Senior Software Engineer, Onsite [Real-time] \n",
|
|
||||||
"24 zip_recruiter Senior Software Engineer - TS/SCI **Minimum $2... \n",
|
|
||||||
"25 zip_recruiter Software Engineer III (full stack, AI/ML, Djan... \n",
|
|
||||||
"26 zip_recruiter Software Engineer Full Stack \n",
|
|
||||||
"27 zip_recruiter Embedded Software Engineer (Fort Worth, TX or ... \n",
|
|
||||||
"28 zip_recruiter Senior Software Engineer (FT) \n",
|
|
||||||
"29 zip_recruiter 2024 Next Gen IT Program | Software Engineerin... \n",
|
|
||||||
"\n",
|
|
||||||
" company_name city state \\\n",
|
|
||||||
"0 Sandstone Care Broomfield CO \n",
|
|
||||||
"1 Noir Consulting Irving TX \n",
|
|
||||||
"2 Johns Hopkins Applied Physics Laboratory (APL) Laurel MD \n",
|
|
||||||
"3 Verkada San Mateo CA \n",
|
|
||||||
"4 Adobe San Jose CA \n",
|
|
||||||
"5 Comcast Philadelphia PA \n",
|
|
||||||
"6 Smart City Solutions FL \n",
|
|
||||||
"7 Honeywell None \n",
|
|
||||||
"8 Fidelity Investments Westlake TX \n",
|
|
||||||
"9 R-DEX Systems, Inc. Atlanta GA \n",
|
|
||||||
"10 Fieldguide San Francisco CA \n",
|
|
||||||
"11 Lockheed Martin Sunnyvale CA \n",
|
|
||||||
"12 Lockheed Martin Edwards CA \n",
|
|
||||||
"13 Lockheed Martin Fort Worth TX \n",
|
|
||||||
"14 Lockheed Martin Fort Worth TX \n",
|
|
||||||
"15 Lockheed Martin Fort Worth TX \n",
|
|
||||||
"16 Lockheed Martin Fort Worth TX \n",
|
|
||||||
"17 SpiderOak Austin TX \n",
|
|
||||||
"18 Rain New York NY \n",
|
|
||||||
"19 Nike Portland OR \n",
|
|
||||||
"20 ZipRecruiter Santa Monica CA \n",
|
|
||||||
"21 ZipRecruiter Phoenix AZ \n",
|
|
||||||
"22 OneStaff Medical Omaha NE \n",
|
|
||||||
"23 Raytheon McKinney TX \n",
|
|
||||||
"24 Raytheon Dallas TX \n",
|
|
||||||
"25 Ayahealthcare Remote OR \n",
|
|
||||||
"26 Generac Power Systems Denver CO \n",
|
|
||||||
"27 Kubota Fort Worth TX \n",
|
|
||||||
"28 National Indoor RV Center Lewisville TX \n",
|
|
||||||
"29 Southern Glazer's Wine & Spirits Dallas TX \n",
|
|
||||||
"\n",
|
|
||||||
" job_type interval min_amount max_amount \\\n",
|
|
||||||
"0 fulltime yearly 68000 57500 \n",
|
|
||||||
"1 None yearly 200000 200000 \n",
|
|
||||||
"2 None None None None \n",
|
|
||||||
"3 fulltime yearly 285000 120000 \n",
|
|
||||||
"4 fulltime yearly 142700 73200 \n",
|
|
||||||
"5 fulltime yearly 184663 78789 \n",
|
|
||||||
"6 fulltime yearly 100000 85000 \n",
|
|
||||||
"7 fulltime None None None \n",
|
|
||||||
"8 None None None None \n",
|
|
||||||
"9 fulltime yearly 160000 120000 \n",
|
|
||||||
"10 fulltime yearly None None \n",
|
|
||||||
"11 fulltime yearly None None \n",
|
|
||||||
"12 fulltime yearly None None \n",
|
|
||||||
"13 fulltime yearly None None \n",
|
|
||||||
"14 fulltime yearly None None \n",
|
|
||||||
"15 fulltime yearly None None \n",
|
|
||||||
"16 fulltime yearly None None \n",
|
|
||||||
"17 fulltime yearly None None \n",
|
|
||||||
"18 fulltime yearly None None \n",
|
|
||||||
"19 contract yearly None None \n",
|
|
||||||
"20 fulltime yearly 130000 150000 \n",
|
|
||||||
"21 fulltime yearly 105000 145000 \n",
|
|
||||||
"22 fulltime yearly 60000 110000 \n",
|
|
||||||
"23 fulltime yearly 116000 153000 \n",
|
|
||||||
"24 fulltime yearly 122000 162000 \n",
|
|
||||||
"25 None yearly 156000 165000 \n",
|
|
||||||
"26 fulltime yearly 90000 115000 \n",
|
|
||||||
"27 fulltime yearly 122000 167000 \n",
|
|
||||||
"28 fulltime yearly 125000 0 \n",
|
|
||||||
"29 None yearly 70000 0 \n",
|
|
||||||
"\n",
|
|
||||||
" job_url \\\n",
|
|
||||||
"0 https://www.indeed.com/viewjob?jk=f5f33d72e030... \n",
|
|
||||||
"1 https://www.indeed.com/viewjob?jk=1b22ba65296c... \n",
|
|
||||||
"2 https://www.indeed.com/viewjob?jk=309eed270a88... \n",
|
|
||||||
"3 https://www.indeed.com/viewjob?jk=a3ea45daca75... \n",
|
|
||||||
"4 https://www.indeed.com/viewjob?jk=0f2dc9901fc7... \n",
|
|
||||||
"5 https://www.indeed.com/viewjob?jk=eb5c927221eb... \n",
|
|
||||||
"6 https://www.indeed.com/viewjob?jk=ba1945f143a1... \n",
|
|
||||||
"7 https://www.indeed.com/viewjob?jk=5a1da623ee75... \n",
|
|
||||||
"8 https://www.indeed.com/viewjob?jk=b600392166bb... \n",
|
|
||||||
"9 https://www.indeed.com/viewjob?jk=a7e9d356c333... \n",
|
|
||||||
"10 https://www.linkedin.com/jobs/view/3696158160 \n",
|
|
||||||
"11 https://www.linkedin.com/jobs/view/3693012711 \n",
|
|
||||||
"12 https://www.linkedin.com/jobs/view/3700669785 \n",
|
|
||||||
"13 https://www.linkedin.com/jobs/view/3701770659 \n",
|
|
||||||
"14 https://www.linkedin.com/jobs/view/3701769637 \n",
|
|
||||||
"15 https://www.linkedin.com/jobs/view/3701772329 \n",
|
|
||||||
"16 https://www.linkedin.com/jobs/view/3701775201 \n",
|
|
||||||
"17 https://www.linkedin.com/jobs/view/3707174719 \n",
|
|
||||||
"18 https://www.linkedin.com/jobs/view/3696158877 \n",
|
|
||||||
"19 https://www.linkedin.com/jobs/view/3693340247 \n",
|
|
||||||
"20 https://www.ziprecruiter.com/c/ZipRecruiter/Jo... \n",
|
|
||||||
"21 https://www.ziprecruiter.com/c/ZipRecruiter/Jo... \n",
|
|
||||||
"22 https://www.ziprecruiter.com/c/OneStaff-Medica... \n",
|
|
||||||
"23 https://jsv3.recruitics.com/redirect?rx_cid=34... \n",
|
|
||||||
"24 https://jsv3.recruitics.com/redirect?rx_cid=34... \n",
|
|
||||||
"25 https://click.appcast.io/track/hcbh0qq?cs=ngp&... \n",
|
|
||||||
"26 https://www.ziprecruiter.com/c/Generac-Power-S... \n",
|
|
||||||
"27 https://us62e2.dayforcehcm.com/CandidatePortal... \n",
|
|
||||||
"28 https://www.ziprecruiter.com/c/National-Indoor... \n",
|
|
||||||
"29 https://click.appcast.io/track/hdsbnae?cs=b4&j... \n",
|
|
||||||
"\n",
|
|
||||||
" description \n",
|
|
||||||
"0 Mental Health Therapist- Broomfield, CO Locati... \n",
|
|
||||||
"1 .NET Software Engineer, C#, WPF - Irving (Tech... \n",
|
|
||||||
"2 Description Are you a communications systems d... \n",
|
|
||||||
"3 Who We Are Verkada is the largest cloud-based ... \n",
|
|
||||||
"4 Our Company Changing the world through digital... \n",
|
|
||||||
"5 Make your mark at Comcast - a Fortune 30 globa... \n",
|
|
||||||
"6 Smart City hiring a full stack software develo... \n",
|
|
||||||
"7 Join a team recognized for leadership, innovat... \n",
|
|
||||||
"8 Job Description: Software Engineer in Test The... \n",
|
|
||||||
"9 Title: Senior DSP/FPGA Firmware Engineer Descr... \n",
|
|
||||||
"10 About us:Fieldguide is establishing a new stat... \n",
|
|
||||||
"11 Description:By bringing together people that u... \n",
|
|
||||||
"12 Description:By bringing together people that u... \n",
|
|
||||||
"13 Description:By bringing together people that u... \n",
|
|
||||||
"14 Description:By bringing together people that u... \n",
|
|
||||||
"15 Description:By bringing together people that u... \n",
|
|
||||||
"16 Description:By bringing together people that u... \n",
|
|
||||||
"17 We're only as strong as our weakest link.In th... \n",
|
|
||||||
"18 Rain’s mission is to create the fastest and ea... \n",
|
|
||||||
"19 Work options: FlexibleWe consider remote, on-p... \n",
|
|
||||||
"20 Demonstrated foundation in software engineerin... \n",
|
|
||||||
"21 Experience in client side development using Re... \n",
|
|
||||||
"22 We are looking for a well-rounded Software Dev... \n",
|
|
||||||
"23 By joining the Silent Knight team as a Senior ... \n",
|
|
||||||
"24 Object Oriented Programming using C++ with Lin... \n",
|
|
||||||
"25 The Software Engineer III will be an integral ... \n",
|
|
||||||
"26 As a Software Engineer on the Energy Technolog... \n",
|
|
||||||
"27 Work with a cross-functional team to design, t... \n",
|
|
||||||
"28 As a Senior Software Engineer, you will: * Des... \n",
|
|
||||||
"29 Finally, through the work assigned, the analys... "
|
|
||||||
]
|
|
||||||
},
|
|
||||||
"execution_count": 5,
|
|
||||||
"metadata": {},
|
|
||||||
"output_type": "execute_result"
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"source": [
|
"source": [
|
||||||
"scrape_jobs(\n",
|
"# example 1 (no hyperlinks, USA)\n",
|
||||||
" site_name=[\"indeed\", \"linkedin\", \"zip_recruiter\"],\n",
|
"jobs = scrape_jobs(\n",
|
||||||
|
" site_name=[\"linkedin\", \"zip_recruiter\"],\n",
|
||||||
|
" location='san francisco',\n",
|
||||||
|
" search_term=\"engineer\",\n",
|
||||||
|
" results_wanted=5,\n",
|
||||||
|
"\n",
|
||||||
|
" # use if you want to use a proxy\n",
|
||||||
|
" # proxy=\"socks5://jobspy:5a4vpWtj8EeJ2hoYzk@ca.smartproxy.com:20001\",\n",
|
||||||
|
" # proxy=\"http://jobspy:5a4vpWtj8EeJ2hoYzk@ca.smartproxy.com:20001\",\n",
|
||||||
|
" # proxy=\"https://jobspy:5a4vpWtj8EeJ2hoYzk@ca.smartproxy.com:20001\",\n",
|
||||||
|
"\n",
|
||||||
|
")\n",
|
||||||
|
"\n",
|
||||||
|
"display(jobs)"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"id": "6a581b2d-f7da-4fac-868d-9efe143ee20a",
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"# example 2 - remote USA & hyperlinks\n",
|
||||||
|
"jobs = scrape_jobs(\n",
|
||||||
|
" site_name=[\"linkedin\", \"zip_recruiter\", \"indeed\"],\n",
|
||||||
|
" # location='san francisco',\n",
|
||||||
" search_term=\"software engineer\",\n",
|
" search_term=\"software engineer\",\n",
|
||||||
" results_wanted=10\n",
|
" country_indeed=\"USA\",\n",
|
||||||
|
" hyperlinks=True,\n",
|
||||||
|
" is_remote=True,\n",
|
||||||
|
" results_wanted=5, \n",
|
||||||
")"
|
")"
|
||||||
]
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"id": "fe8289bc-5b64-4202-9a64-7c117c83fd9a",
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"# use if hyperlinks=True\n",
|
||||||
|
"html = jobs.to_html(escape=False)\n",
|
||||||
|
"# change max-width: 200px to show more or less of the content\n",
|
||||||
|
"truncate_width = f'<style>.dataframe td {{ max-width: 200px; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; }}</style>{html}'\n",
|
||||||
|
"display(HTML(truncate_width))"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"id": "951c2fe1-52ff-407d-8bb1-068049b36777",
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"# example 3 - with hyperlinks, international - linkedin (no zip_recruiter)\n",
|
||||||
|
"jobs = scrape_jobs(\n",
|
||||||
|
" site_name=[\"linkedin\"],\n",
|
||||||
|
" location='berlin',\n",
|
||||||
|
" search_term=\"engineer\",\n",
|
||||||
|
" hyperlinks=True,\n",
|
||||||
|
" results_wanted=5,\n",
|
||||||
|
" easy_apply=True\n",
|
||||||
|
"\n",
|
||||||
|
"\n",
|
||||||
|
"\n",
|
||||||
|
")"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"id": "1e37a521-caef-441c-8fc2-2eb5b2e7da62",
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"# use if hyperlinks=True\n",
|
||||||
|
"html = jobs.to_html(escape=False)\n",
|
||||||
|
"# change max-width: 200px to show more or less of the content\n",
|
||||||
|
"truncate_width = f'<style>.dataframe td {{ max-width: 200px; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; }}</style>{html}'\n",
|
||||||
|
"display(HTML(truncate_width))"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"id": "0650e608-0b58-4bf5-ae86-68348035b16a",
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"# example 4 - international indeed (no zip_recruiter)\n",
|
||||||
|
"result = scrape_jobs(\n",
|
||||||
|
" site_name=[\"indeed\"],\n",
|
||||||
|
" location='berlin',\n",
|
||||||
|
" search_term=\"engineer\",\n",
|
||||||
|
" country_indeed = \"Germany\",\n",
|
||||||
|
" hyperlinks=True\n",
|
||||||
|
")"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"id": "40913ac8-3f8a-4d7e-ac47-afb88316432b",
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"# use if hyperlinks=True\n",
|
||||||
|
"html = jobs.to_html(escape=False)\n",
|
||||||
|
"# change max-width: 200px to show more or less of the content\n",
|
||||||
|
"truncate_width = f'<style>.dataframe td {{ max-width: 200px; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; }}</style>{html}'\n",
|
||||||
|
"display(HTML(truncate_width))"
|
||||||
|
]
|
||||||
}
|
}
|
||||||
],
|
],
|
||||||
"metadata": {
|
"metadata": {
|
||||||
@@ -681,7 +165,7 @@
|
|||||||
"name": "python",
|
"name": "python",
|
||||||
"nbconvert_exporter": "python",
|
"nbconvert_exporter": "python",
|
||||||
"pygments_lexer": "ipython3",
|
"pygments_lexer": "ipython3",
|
||||||
"version": "3.11.4"
|
"version": "3.10.11"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"nbformat": 4,
|
"nbformat": 4,
|
||||||
|
|||||||
101
README.md
101
README.md
@@ -1,15 +1,22 @@
|
|||||||
# <img src="https://github.com/cullenwatson/JobSpy/assets/78247585/2f61a059-9647-4a9c-bfb9-e3a9448bdc6a" style="vertical-align: sub; margin-right: 5px;"> JobSpy
|
<img src="https://github.com/cullenwatson/JobSpy/assets/78247585/ae185b7e-e444-4712-8bb9-fa97f53e896b" width="400">
|
||||||
|
|
||||||
**JobSpy** is a simple, yet comprehensive, job scraping library.
|
**JobSpy** is a simple, yet comprehensive, job scraping library.
|
||||||
## Features
|
## Features
|
||||||
|
|
||||||
|
|
||||||
- Scrapes job postings from **LinkedIn**, **Indeed** & **ZipRecruiter** simultaneously
|
- Scrapes job postings from **LinkedIn**, **Indeed** & **ZipRecruiter** simultaneously
|
||||||
- Aggregates the job postings in a Pandas DataFrame
|
- Aggregates the job postings in a Pandas DataFrame
|
||||||
|
|
||||||
|
[Video Guide for JobSpy](https://www.youtube.com/watch?v=-yS3mgI5H-4)
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
### Installation
|
### Installation
|
||||||
`pip install python-jobspy`
|
```
|
||||||
|
pip install python-jobspy
|
||||||
|
```
|
||||||
|
|
||||||
_Python version >= [3.10](https://www.python.org/downloads/release/python-3100/) required_
|
_Python version >= [3.10](https://www.python.org/downloads/release/python-3100/) required_
|
||||||
|
|
||||||
@@ -22,24 +29,32 @@ import pandas as pd
|
|||||||
jobs: pd.DataFrame = scrape_jobs(
|
jobs: pd.DataFrame = scrape_jobs(
|
||||||
site_name=["indeed", "linkedin", "zip_recruiter"],
|
site_name=["indeed", "linkedin", "zip_recruiter"],
|
||||||
search_term="software engineer",
|
search_term="software engineer",
|
||||||
results_wanted=10
|
location="Dallas, TX",
|
||||||
|
results_wanted=10,
|
||||||
|
|
||||||
|
country_indeed='USA' # only needed for indeed
|
||||||
|
|
||||||
|
# use if you want to use a proxy
|
||||||
|
# proxy="socks5://jobspy:5a4vpWtj8EeJ2hoYzk@ca.smartproxy.com:20001",
|
||||||
|
# proxy="http://jobspy:5a4vpWtj8EeJ2hoYzk@ca.smartproxy.com:20001",
|
||||||
|
# proxy="https://jobspy:5a4vpWtj8EeJ2hoYzk@ca.smartproxy.com:20001",
|
||||||
)
|
)
|
||||||
|
|
||||||
if jobs.empty:
|
pd.set_option('display.max_columns', None)
|
||||||
print("No jobs found.")
|
pd.set_option('display.max_rows', None)
|
||||||
else:
|
pd.set_option('display.width', None)
|
||||||
#1 print
|
pd.set_option('display.max_colwidth', 50) # set to 0 to see full job url / desc
|
||||||
pd.set_option('display.max_columns', None)
|
|
||||||
pd.set_option('display.max_rows', None)
|
|
||||||
pd.set_option('display.width', None)
|
|
||||||
pd.set_option('display.max_colwidth', 50) # set to 0 to see full job url / desc
|
|
||||||
print(jobs)
|
|
||||||
|
|
||||||
#2 display in Jupyter Notebook
|
#1 output
|
||||||
#display(jobs)
|
print(jobs)
|
||||||
|
print(errors)
|
||||||
|
|
||||||
#3 output to .csv
|
#2 display in Jupyter Notebook
|
||||||
#jobs.to_csv('jobs.csv', index=False)
|
#display(jobs)
|
||||||
|
#display(errors)
|
||||||
|
|
||||||
|
#3 output to .csv
|
||||||
|
#result.jobs.to_csv('result.jobs.csv', index=False)
|
||||||
```
|
```
|
||||||
|
|
||||||
### Output
|
### Output
|
||||||
@@ -63,29 +78,67 @@ Optional
|
|||||||
├── job_type (enum): fulltime, parttime, internship, contract
|
├── job_type (enum): fulltime, parttime, internship, contract
|
||||||
├── is_remote (bool)
|
├── is_remote (bool)
|
||||||
├── results_wanted (int): number of job results to retrieve for each site specified in 'site_type'
|
├── results_wanted (int): number of job results to retrieve for each site specified in 'site_type'
|
||||||
├── easy_apply (bool): filters for jobs on LinkedIn that have the 'Easy Apply' option
|
├── easy_apply (bool): filters for jobs that are hosted on LinkedIn
|
||||||
|
├── country_indeed (enum): filters the country on Indeed
|
||||||
```
|
```
|
||||||
|
|
||||||
|
|
||||||
### JobPost Schema
|
### JobPost Schema
|
||||||
```plaintext
|
```plaintext
|
||||||
JobPost
|
JobPost
|
||||||
├── title (str)
|
├── title (str)
|
||||||
├── company_name (str)
|
├── company (str)
|
||||||
├── job_url (str)
|
├── job_url (str)
|
||||||
├── location (object)
|
├── location (object)
|
||||||
│ ├── country (str)
|
│ ├── country (str)
|
||||||
│ ├── city (str)
|
│ ├── city (str)
|
||||||
│ ├── state (str)
|
│ ├── state (str)
|
||||||
├── description (str)
|
├── description (str)
|
||||||
├── job_type (enum)
|
├── job_type (enum): fulltime, parttime, internship, contract
|
||||||
├── compensation (object)
|
├── compensation (object)
|
||||||
│ ├── interval (CompensationInterval): yearly, monthly, weekly, daily, hourly
|
│ ├── interval (enum): yearly, monthly, weekly, daily, hourly
|
||||||
│ ├── min_amount (float)
|
│ ├── min_amount (int)
|
||||||
│ ├── max_amount (float)
|
│ ├── max_amount (int)
|
||||||
│ └── currency (str)
|
│ └── currency (enum)
|
||||||
└── date_posted (datetime)
|
└── date_posted (date)
|
||||||
```
|
```
|
||||||
|
|
||||||
|
## Supported Countries for Job Searching
|
||||||
|
|
||||||
|
|
||||||
|
### **LinkedIn**
|
||||||
|
|
||||||
|
LinkedIn searches globally & uses only the `location` parameter
|
||||||
|
|
||||||
|
### **ZipRecruiter**
|
||||||
|
|
||||||
|
ZipRecruiter searches for jobs in US/Canada & uses only the `location` parameter
|
||||||
|
|
||||||
|
|
||||||
|
### **Indeed**
|
||||||
|
For Indeed, the `country_indeed` parameter is required. Additionally, use the `location` parameter and include the city or state if necessary.
|
||||||
|
|
||||||
|
You can specify the following countries when searching on Indeed (use the exact name):
|
||||||
|
|
||||||
|
|
||||||
|
| | | | |
|
||||||
|
|------|------|------|------|
|
||||||
|
| Argentina | Australia | Austria | Bahrain |
|
||||||
|
| Belgium | Brazil | Canada | Chile |
|
||||||
|
| China | Colombia | Costa Rica | Czech Republic |
|
||||||
|
| Denmark | Ecuador | Egypt | Finland |
|
||||||
|
| France | Germany | Greece | Hong Kong |
|
||||||
|
| Hungary | India | Indonesia | Ireland |
|
||||||
|
| Israel | Italy | Japan | Kuwait |
|
||||||
|
| Luxembourg | Malaysia | Mexico | Morocco |
|
||||||
|
| Netherlands | New Zealand | Nigeria | Norway |
|
||||||
|
| Oman | Pakistan | Panama | Peru |
|
||||||
|
| Philippines | Poland | Portugal | Qatar |
|
||||||
|
| Romania | Saudi Arabia | Singapore | South Africa |
|
||||||
|
| South Korea | Spain | Sweden | Switzerland |
|
||||||
|
| Taiwan | Thailand | Turkey | Ukraine |
|
||||||
|
| United Arab Emirates | UK | USA | Uruguay |
|
||||||
|
| Venezuela | Vietnam | | |
|
||||||
|
|
||||||
## Frequently Asked Questions
|
## Frequently Asked Questions
|
||||||
|
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
[tool.poetry]
|
[tool.poetry]
|
||||||
name = "python-jobspy"
|
name = "python-jobspy"
|
||||||
version = "1.0.3"
|
version = "1.1.3"
|
||||||
description = "Job scraper for LinkedIn, Indeed & ZipRecruiter"
|
description = "Job scraper for LinkedIn, Indeed & ZipRecruiter"
|
||||||
authors = ["Zachary Hampton <zachary@zacharysproducts.com>", "Cullen Watson <cullen@cullen.ai>"]
|
authors = ["Zachary Hampton <zachary@zacharysproducts.com>", "Cullen Watson <cullen@cullen.ai>"]
|
||||||
readme = "README.md"
|
readme = "README.md"
|
||||||
|
|||||||
@@ -1,17 +1,20 @@
|
|||||||
import pandas as pd
|
import pandas as pd
|
||||||
from typing import List, Tuple
|
import concurrent.futures
|
||||||
|
from concurrent.futures import ThreadPoolExecutor
|
||||||
|
from typing import List, Tuple, NamedTuple, Dict, Optional
|
||||||
|
import traceback
|
||||||
|
|
||||||
from .jobs import JobType
|
from .jobs import JobType, Location
|
||||||
from .scrapers.indeed import IndeedScraper
|
from .scrapers.indeed import IndeedScraper
|
||||||
from .scrapers.ziprecruiter import ZipRecruiterScraper
|
from .scrapers.ziprecruiter import ZipRecruiterScraper
|
||||||
from .scrapers.linkedin import LinkedInScraper
|
from .scrapers.linkedin import LinkedInScraper
|
||||||
from .scrapers import (
|
from .scrapers import ScraperInput, Site, JobResponse, Country
|
||||||
ScraperInput,
|
from .scrapers.exceptions import (
|
||||||
Site,
|
LinkedInException,
|
||||||
JobResponse,
|
IndeedException,
|
||||||
|
ZipRecruiterException,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
SCRAPER_MAPPING = {
|
SCRAPER_MAPPING = {
|
||||||
Site.LINKEDIN: LinkedInScraper,
|
Site.LINKEDIN: LinkedInScraper,
|
||||||
Site.INDEED: IndeedScraper,
|
Site.INDEED: IndeedScraper,
|
||||||
@@ -24,7 +27,7 @@ def _map_str_to_site(site_name: str) -> Site:
|
|||||||
|
|
||||||
|
|
||||||
def scrape_jobs(
|
def scrape_jobs(
|
||||||
site_name: str | Site | List[Site],
|
site_name: str | List[str] | Site | List[Site],
|
||||||
search_term: str,
|
search_term: str,
|
||||||
location: str = "",
|
location: str = "",
|
||||||
distance: int = None,
|
distance: int = None,
|
||||||
@@ -32,18 +35,28 @@ def scrape_jobs(
|
|||||||
job_type: JobType = None,
|
job_type: JobType = None,
|
||||||
easy_apply: bool = False, # linkedin
|
easy_apply: bool = False, # linkedin
|
||||||
results_wanted: int = 15,
|
results_wanted: int = 15,
|
||||||
|
country_indeed: str = "usa",
|
||||||
|
hyperlinks: bool = False,
|
||||||
|
proxy: Optional[str] = None,
|
||||||
) -> pd.DataFrame:
|
) -> pd.DataFrame:
|
||||||
"""
|
"""
|
||||||
Asynchronously scrapes job data from multiple job sites.
|
Simultaneously scrapes job data from multiple job sites.
|
||||||
:return: results_wanted: pandas dataframe containing job data
|
:return: results_wanted: pandas dataframe containing job data
|
||||||
"""
|
"""
|
||||||
|
|
||||||
if type(site_name) == str:
|
if type(site_name) == str:
|
||||||
site_name = _map_str_to_site(site_name)
|
site_type = [_map_str_to_site(site_name)]
|
||||||
|
else: #: if type(site_name) == list
|
||||||
|
site_type = [
|
||||||
|
_map_str_to_site(site) if type(site) == str else site_name
|
||||||
|
for site in site_name
|
||||||
|
]
|
||||||
|
|
||||||
|
country_enum = Country.from_string(country_indeed)
|
||||||
|
|
||||||
site_type = [site_name] if type(site_name) == Site else site_name
|
|
||||||
scraper_input = ScraperInput(
|
scraper_input = ScraperInput(
|
||||||
site_type=site_type,
|
site_type=site_type,
|
||||||
|
country=country_enum,
|
||||||
search_term=search_term,
|
search_term=search_term,
|
||||||
location=location,
|
location=location,
|
||||||
distance=distance,
|
distance=distance,
|
||||||
@@ -55,74 +68,94 @@ def scrape_jobs(
|
|||||||
|
|
||||||
def scrape_site(site: Site) -> Tuple[str, JobResponse]:
|
def scrape_site(site: Site) -> Tuple[str, JobResponse]:
|
||||||
scraper_class = SCRAPER_MAPPING[site]
|
scraper_class = SCRAPER_MAPPING[site]
|
||||||
scraper = scraper_class()
|
scraper = scraper_class(proxy=proxy)
|
||||||
scraped_data: JobResponse = scraper.scrape(scraper_input)
|
|
||||||
|
|
||||||
|
try:
|
||||||
|
scraped_data: JobResponse = scraper.scrape(scraper_input)
|
||||||
|
except (LinkedInException, IndeedException, ZipRecruiterException) as lie:
|
||||||
|
raise lie
|
||||||
|
except Exception as e:
|
||||||
|
# unhandled exceptions
|
||||||
|
if site == Site.LINKEDIN:
|
||||||
|
raise LinkedInException()
|
||||||
|
if site == Site.INDEED:
|
||||||
|
raise IndeedException()
|
||||||
|
if site == Site.ZIP_RECRUITER:
|
||||||
|
raise ZipRecruiterException()
|
||||||
|
else:
|
||||||
|
raise e
|
||||||
return site.value, scraped_data
|
return site.value, scraped_data
|
||||||
|
|
||||||
results = {}
|
site_to_jobs_dict = {}
|
||||||
for site in scraper_input.site_type:
|
|
||||||
|
def worker(site):
|
||||||
site_value, scraped_data = scrape_site(site)
|
site_value, scraped_data = scrape_site(site)
|
||||||
results[site_value] = scraped_data
|
return site_value, scraped_data
|
||||||
|
|
||||||
dfs = []
|
with ThreadPoolExecutor() as executor:
|
||||||
|
future_to_site = {
|
||||||
|
executor.submit(worker, site): site for site in scraper_input.site_type
|
||||||
|
}
|
||||||
|
|
||||||
for site, job_response in results.items():
|
for future in concurrent.futures.as_completed(future_to_site):
|
||||||
|
site_value, scraped_data = future.result()
|
||||||
|
site_to_jobs_dict[site_value] = scraped_data
|
||||||
|
|
||||||
|
jobs_dfs: List[pd.DataFrame] = []
|
||||||
|
|
||||||
|
for site, job_response in site_to_jobs_dict.items():
|
||||||
for job in job_response.jobs:
|
for job in job_response.jobs:
|
||||||
data = job.dict()
|
job_data = job.dict()
|
||||||
data["site"] = site
|
job_data[
|
||||||
|
"job_url_hyper"
|
||||||
# Formatting JobType
|
] = f'<a href="{job_data["job_url"]}">{job_data["job_url"]}</a>'
|
||||||
data["job_type"] = data["job_type"].value if data["job_type"] else None
|
job_data["site"] = site
|
||||||
|
job_data["company"] = job_data["company_name"]
|
||||||
# Formatting Location
|
if job_data["job_type"]:
|
||||||
location_obj = data.get("location")
|
# Take the first value from the job type tuple
|
||||||
if location_obj and isinstance(location_obj, dict):
|
job_data["job_type"] = job_data["job_type"].value[0]
|
||||||
data["city"] = location_obj.get("city", "")
|
|
||||||
data["state"] = location_obj.get("state", "")
|
|
||||||
data["country"] = location_obj.get("country", "USA")
|
|
||||||
else:
|
else:
|
||||||
data["city"] = None
|
job_data["job_type"] = None
|
||||||
data["state"] = None
|
|
||||||
data["country"] = None
|
|
||||||
|
|
||||||
# Formatting Compensation
|
job_data["location"] = Location(**job_data["location"]).display_location()
|
||||||
compensation_obj = data.get("compensation")
|
|
||||||
|
compensation_obj = job_data.get("compensation")
|
||||||
if compensation_obj and isinstance(compensation_obj, dict):
|
if compensation_obj and isinstance(compensation_obj, dict):
|
||||||
data["interval"] = (
|
job_data["interval"] = (
|
||||||
compensation_obj.get("interval").value
|
compensation_obj.get("interval").value
|
||||||
if compensation_obj.get("interval")
|
if compensation_obj.get("interval")
|
||||||
else None
|
else None
|
||||||
)
|
)
|
||||||
data["min_amount"] = compensation_obj.get("min_amount")
|
job_data["min_amount"] = compensation_obj.get("min_amount")
|
||||||
data["max_amount"] = compensation_obj.get("max_amount")
|
job_data["max_amount"] = compensation_obj.get("max_amount")
|
||||||
data["currency"] = compensation_obj.get("currency", "USD")
|
job_data["currency"] = compensation_obj.get("currency", "USD")
|
||||||
else:
|
else:
|
||||||
data["interval"] = None
|
job_data["interval"] = None
|
||||||
data["min_amount"] = None
|
job_data["min_amount"] = None
|
||||||
data["max_amount"] = None
|
job_data["max_amount"] = None
|
||||||
data["currency"] = None
|
job_data["currency"] = None
|
||||||
|
|
||||||
job_df = pd.DataFrame([data])
|
job_df = pd.DataFrame([job_data])
|
||||||
dfs.append(job_df)
|
jobs_dfs.append(job_df)
|
||||||
|
|
||||||
if dfs:
|
if jobs_dfs:
|
||||||
df = pd.concat(dfs, ignore_index=True)
|
jobs_df = pd.concat(jobs_dfs, ignore_index=True)
|
||||||
desired_order = [
|
desired_order: List[str] = [
|
||||||
"site",
|
"site",
|
||||||
"title",
|
"title",
|
||||||
"company_name",
|
"company",
|
||||||
"city",
|
"location",
|
||||||
"state",
|
"date_posted",
|
||||||
"job_type",
|
"job_type",
|
||||||
"interval",
|
"interval",
|
||||||
"min_amount",
|
"min_amount",
|
||||||
"max_amount",
|
"max_amount",
|
||||||
"job_url",
|
"currency",
|
||||||
|
"job_url_hyper" if hyperlinks else "job_url",
|
||||||
"description",
|
"description",
|
||||||
]
|
]
|
||||||
df = df[desired_order]
|
jobs_formatted_df = jobs_df[desired_order]
|
||||||
else:
|
else:
|
||||||
df = pd.DataFrame()
|
jobs_formatted_df = pd.DataFrame()
|
||||||
|
|
||||||
return df
|
return jobs_formatted_df
|
||||||
|
|||||||
@@ -6,24 +6,160 @@ from pydantic import BaseModel, validator
|
|||||||
|
|
||||||
|
|
||||||
class JobType(Enum):
|
class JobType(Enum):
|
||||||
FULL_TIME = "fulltime"
|
FULL_TIME = (
|
||||||
PART_TIME = "parttime"
|
"fulltime",
|
||||||
CONTRACT = "contract"
|
"períodointegral",
|
||||||
TEMPORARY = "temporary"
|
"estágio/trainee",
|
||||||
INTERNSHIP = "internship"
|
"cunormăîntreagă",
|
||||||
|
"tiempocompleto",
|
||||||
|
"vollzeit",
|
||||||
|
"voltijds",
|
||||||
|
"tempointegral",
|
||||||
|
"全职",
|
||||||
|
"plnýúvazek",
|
||||||
|
"fuldtid",
|
||||||
|
"دوامكامل",
|
||||||
|
"kokopäivätyö",
|
||||||
|
"tempsplein",
|
||||||
|
"vollzeit",
|
||||||
|
"πλήρηςαπασχόληση",
|
||||||
|
"teljesmunkaidő",
|
||||||
|
"tempopieno",
|
||||||
|
"tempsplein",
|
||||||
|
"heltid",
|
||||||
|
"jornadacompleta",
|
||||||
|
"pełnyetat",
|
||||||
|
"정규직",
|
||||||
|
"100%",
|
||||||
|
"全職",
|
||||||
|
"งานประจำ",
|
||||||
|
"tamzamanlı",
|
||||||
|
"повназайнятість",
|
||||||
|
"toànthờigian",
|
||||||
|
)
|
||||||
|
PART_TIME = ("parttime", "teilzeit")
|
||||||
|
CONTRACT = ("contract", "contractor")
|
||||||
|
TEMPORARY = ("temporary",)
|
||||||
|
INTERNSHIP = ("internship", "prácticas", "ojt(onthejobtraining)", "praktikum")
|
||||||
|
|
||||||
PER_DIEM = "perdiem"
|
PER_DIEM = ("perdiem",)
|
||||||
NIGHTS = "nights"
|
NIGHTS = ("nights",)
|
||||||
OTHER = "other"
|
OTHER = ("other",)
|
||||||
SUMMER = "summer"
|
SUMMER = ("summer",)
|
||||||
VOLUNTEER = "volunteer"
|
VOLUNTEER = ("volunteer",)
|
||||||
|
|
||||||
|
|
||||||
|
class Country(Enum):
|
||||||
|
ARGENTINA = ("argentina", "ar")
|
||||||
|
AUSTRALIA = ("australia", "au")
|
||||||
|
AUSTRIA = ("austria", "at")
|
||||||
|
BAHRAIN = ("bahrain", "bh")
|
||||||
|
BELGIUM = ("belgium", "be")
|
||||||
|
BRAZIL = ("brazil", "br")
|
||||||
|
CANADA = ("canada", "ca")
|
||||||
|
CHILE = ("chile", "cl")
|
||||||
|
CHINA = ("china", "cn")
|
||||||
|
COLOMBIA = ("colombia", "co")
|
||||||
|
COSTARICA = ("costa rica", "cr")
|
||||||
|
CZECHREPUBLIC = ("czech republic", "cz")
|
||||||
|
DENMARK = ("denmark", "dk")
|
||||||
|
ECUADOR = ("ecuador", "ec")
|
||||||
|
EGYPT = ("egypt", "eg")
|
||||||
|
FINLAND = ("finland", "fi")
|
||||||
|
FRANCE = ("france", "fr")
|
||||||
|
GERMANY = ("germany", "de")
|
||||||
|
GREECE = ("greece", "gr")
|
||||||
|
HONGKONG = ("hong kong", "hk")
|
||||||
|
HUNGARY = ("hungary", "hu")
|
||||||
|
INDIA = ("india", "in")
|
||||||
|
INDONESIA = ("indonesia", "id")
|
||||||
|
IRELAND = ("ireland", "ie")
|
||||||
|
ISRAEL = ("israel", "il")
|
||||||
|
ITALY = ("italy", "it")
|
||||||
|
JAPAN = ("japan", "jp")
|
||||||
|
KUWAIT = ("kuwait", "kw")
|
||||||
|
LUXEMBOURG = ("luxembourg", "lu")
|
||||||
|
MALAYSIA = ("malaysia", "malaysia")
|
||||||
|
MEXICO = ("mexico", "mx")
|
||||||
|
MOROCCO = ("morocco", "ma")
|
||||||
|
NETHERLANDS = ("netherlands", "nl")
|
||||||
|
NEWZEALAND = ("new zealand", "nz")
|
||||||
|
NIGERIA = ("nigeria", "ng")
|
||||||
|
NORWAY = ("norway", "no")
|
||||||
|
OMAN = ("oman", "om")
|
||||||
|
PAKISTAN = ("pakistan", "pk")
|
||||||
|
PANAMA = ("panama", "pa")
|
||||||
|
PERU = ("peru", "pe")
|
||||||
|
PHILIPPINES = ("philippines", "ph")
|
||||||
|
POLAND = ("poland", "pl")
|
||||||
|
PORTUGAL = ("portugal", "pt")
|
||||||
|
QATAR = ("qatar", "qa")
|
||||||
|
ROMANIA = ("romania", "ro")
|
||||||
|
SAUDIARABIA = ("saudi arabia", "sa")
|
||||||
|
SINGAPORE = ("singapore", "sg")
|
||||||
|
SOUTHAFRICA = ("south africa", "za")
|
||||||
|
SOUTHKOREA = ("south korea", "kr")
|
||||||
|
SPAIN = ("spain", "es")
|
||||||
|
SWEDEN = ("sweden", "se")
|
||||||
|
SWITZERLAND = ("switzerland", "ch")
|
||||||
|
TAIWAN = ("taiwan", "tw")
|
||||||
|
THAILAND = ("thailand", "th")
|
||||||
|
TURKEY = ("turkey", "tr")
|
||||||
|
UKRAINE = ("ukraine", "ua")
|
||||||
|
UNITEDARABEMIRATES = ("united arab emirates", "ae")
|
||||||
|
UK = ("uk", "uk")
|
||||||
|
USA = ("usa", "www")
|
||||||
|
URUGUAY = ("uruguay", "uy")
|
||||||
|
VENEZUELA = ("venezuela", "ve")
|
||||||
|
VIETNAM = ("vietnam", "vn")
|
||||||
|
|
||||||
|
# internal for ziprecruiter
|
||||||
|
US_CANADA = ("usa/ca", "www")
|
||||||
|
|
||||||
|
# internal for linkeind
|
||||||
|
WORLDWIDE = ("worldwide", "www")
|
||||||
|
|
||||||
|
def __new__(cls, country, domain):
|
||||||
|
obj = object.__new__(cls)
|
||||||
|
obj._value_ = country
|
||||||
|
obj.domain = domain
|
||||||
|
return obj
|
||||||
|
|
||||||
|
@property
|
||||||
|
def domain_value(self):
|
||||||
|
return self.domain
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def from_string(cls, country_str: str):
|
||||||
|
"""Convert a string to the corresponding Country enum."""
|
||||||
|
country_str = country_str.strip().lower()
|
||||||
|
for country in cls:
|
||||||
|
if country.value == country_str:
|
||||||
|
return country
|
||||||
|
valid_countries = [country.value for country in cls]
|
||||||
|
raise ValueError(
|
||||||
|
f"Invalid country string: '{country_str}'. Valid countries (only include this param for Indeed) are: {', '.join(valid_countries)}"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class Location(BaseModel):
|
class Location(BaseModel):
|
||||||
country: str = "USA"
|
country: Country = None
|
||||||
city: str = None
|
city: Optional[str] = None
|
||||||
state: Optional[str] = None
|
state: Optional[str] = None
|
||||||
|
|
||||||
|
def display_location(self) -> str:
|
||||||
|
location_parts = []
|
||||||
|
if self.city:
|
||||||
|
location_parts.append(self.city)
|
||||||
|
if self.state:
|
||||||
|
location_parts.append(self.state)
|
||||||
|
if self.country and self.country not in (Country.US_CANADA, Country.WORLDWIDE):
|
||||||
|
if self.country.value in ("usa", "uk"):
|
||||||
|
location_parts.append(self.country.value.upper())
|
||||||
|
else:
|
||||||
|
location_parts.append(self.country.value.title())
|
||||||
|
return ", ".join(location_parts)
|
||||||
|
|
||||||
|
|
||||||
class CompensationInterval(Enum):
|
class CompensationInterval(Enum):
|
||||||
YEARLY = "yearly"
|
YEARLY = "yearly"
|
||||||
@@ -37,7 +173,7 @@ class Compensation(BaseModel):
|
|||||||
interval: CompensationInterval
|
interval: CompensationInterval
|
||||||
min_amount: int = None
|
min_amount: int = None
|
||||||
max_amount: int = None
|
max_amount: int = None
|
||||||
currency: str = "USD"
|
currency: Optional[str] = "USD"
|
||||||
|
|
||||||
|
|
||||||
class JobPost(BaseModel):
|
class JobPost(BaseModel):
|
||||||
@@ -53,22 +189,4 @@ class JobPost(BaseModel):
|
|||||||
|
|
||||||
|
|
||||||
class JobResponse(BaseModel):
|
class JobResponse(BaseModel):
|
||||||
success: bool
|
|
||||||
error: str = None
|
|
||||||
|
|
||||||
total_results: Optional[int] = None
|
|
||||||
|
|
||||||
jobs: list[JobPost] = []
|
jobs: list[JobPost] = []
|
||||||
|
|
||||||
returned_results: int = None
|
|
||||||
|
|
||||||
@validator("returned_results", pre=True, always=True)
|
|
||||||
def set_returned_results(cls, v, values):
|
|
||||||
jobs_list = values.get("jobs")
|
|
||||||
|
|
||||||
if v is None:
|
|
||||||
if jobs_list is not None:
|
|
||||||
return len(jobs_list)
|
|
||||||
else:
|
|
||||||
return 0
|
|
||||||
return v
|
|
||||||
|
|||||||
@@ -1,12 +1,7 @@
|
|||||||
from ..jobs import Enum, BaseModel, JobType, JobResponse
|
from ..jobs import Enum, BaseModel, JobType, JobResponse, Country
|
||||||
from typing import List, Optional, Any
|
from typing import List, Optional, Any
|
||||||
|
|
||||||
|
|
||||||
class StatusException(Exception):
|
|
||||||
def __init__(self, status_code: int):
|
|
||||||
self.status_code = status_code
|
|
||||||
|
|
||||||
|
|
||||||
class Site(Enum):
|
class Site(Enum):
|
||||||
LINKEDIN = "linkedin"
|
LINKEDIN = "linkedin"
|
||||||
INDEED = "indeed"
|
INDEED = "indeed"
|
||||||
@@ -18,6 +13,7 @@ class ScraperInput(BaseModel):
|
|||||||
search_term: str
|
search_term: str
|
||||||
|
|
||||||
location: str = None
|
location: str = None
|
||||||
|
country: Optional[Country] = Country.USA
|
||||||
distance: Optional[int] = None
|
distance: Optional[int] = None
|
||||||
is_remote: bool = False
|
is_remote: bool = False
|
||||||
job_type: Optional[JobType] = None
|
job_type: Optional[JobType] = None
|
||||||
@@ -26,18 +22,10 @@ class ScraperInput(BaseModel):
|
|||||||
results_wanted: int = 15
|
results_wanted: int = 15
|
||||||
|
|
||||||
|
|
||||||
class CommonResponse(BaseModel):
|
|
||||||
status: Optional[str]
|
|
||||||
error: Optional[str]
|
|
||||||
linkedin: Optional[Any] = None
|
|
||||||
indeed: Optional[Any] = None
|
|
||||||
zip_recruiter: Optional[Any] = None
|
|
||||||
|
|
||||||
|
|
||||||
class Scraper:
|
class Scraper:
|
||||||
def __init__(self, site: Site, url: str):
|
def __init__(self, site: Site, proxy: Optional[List[str]] = None):
|
||||||
self.site = site
|
self.site = site
|
||||||
self.url = url
|
self.proxy = (lambda p: {"http": p, "https": p} if p else None)(proxy)
|
||||||
|
|
||||||
def scrape(self, scraper_input: ScraperInput) -> JobResponse:
|
def scrape(self, scraper_input: ScraperInput) -> JobResponse:
|
||||||
...
|
...
|
||||||
|
|||||||
18
src/jobspy/scrapers/exceptions.py
Normal file
18
src/jobspy/scrapers/exceptions.py
Normal file
@@ -0,0 +1,18 @@
|
|||||||
|
"""
|
||||||
|
jobspy.scrapers.exceptions
|
||||||
|
~~~~~~~~~~~~~~~~~~~
|
||||||
|
|
||||||
|
This module contains the set of Scrapers' exceptions.
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
|
class LinkedInException(Exception):
|
||||||
|
"""Failed to scrape LinkedIn"""
|
||||||
|
|
||||||
|
|
||||||
|
class IndeedException(Exception):
|
||||||
|
"""Failed to scrape Indeed"""
|
||||||
|
|
||||||
|
|
||||||
|
class ZipRecruiterException(Exception):
|
||||||
|
"""Failed to scrape ZipRecruiter"""
|
||||||
@@ -1,6 +1,14 @@
|
|||||||
|
"""
|
||||||
|
jobspy.scrapers.indeed
|
||||||
|
~~~~~~~~~~~~~~~~~~~
|
||||||
|
|
||||||
|
This module contains routines to scrape Indeed.
|
||||||
|
"""
|
||||||
import re
|
import re
|
||||||
import math
|
import math
|
||||||
|
import io
|
||||||
import json
|
import json
|
||||||
|
import traceback
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
from typing import Optional
|
from typing import Optional
|
||||||
|
|
||||||
@@ -10,6 +18,7 @@ from bs4 import BeautifulSoup
|
|||||||
from bs4.element import Tag
|
from bs4.element import Tag
|
||||||
from concurrent.futures import ThreadPoolExecutor, Future
|
from concurrent.futures import ThreadPoolExecutor, Future
|
||||||
|
|
||||||
|
from ..exceptions import IndeedException
|
||||||
from ...jobs import (
|
from ...jobs import (
|
||||||
JobPost,
|
JobPost,
|
||||||
Compensation,
|
Compensation,
|
||||||
@@ -18,21 +27,16 @@ from ...jobs import (
|
|||||||
JobResponse,
|
JobResponse,
|
||||||
JobType,
|
JobType,
|
||||||
)
|
)
|
||||||
from .. import Scraper, ScraperInput, Site, StatusException
|
from .. import Scraper, ScraperInput, Site, Country
|
||||||
|
|
||||||
|
|
||||||
class ParsingException(Exception):
|
|
||||||
pass
|
|
||||||
|
|
||||||
|
|
||||||
class IndeedScraper(Scraper):
|
class IndeedScraper(Scraper):
|
||||||
def __init__(self):
|
def __init__(self, proxy: Optional[str] = None):
|
||||||
"""
|
"""
|
||||||
Initializes IndeedScraper with the Indeed job search url
|
Initializes IndeedScraper with the Indeed job search url
|
||||||
"""
|
"""
|
||||||
site = Site(Site.INDEED)
|
site = Site(Site.INDEED)
|
||||||
url = "https://www.indeed.com"
|
super().__init__(site, proxy=proxy)
|
||||||
super().__init__(site, url)
|
|
||||||
|
|
||||||
self.jobs_per_page = 15
|
self.jobs_per_page = 15
|
||||||
self.seen_urls = set()
|
self.seen_urls = set()
|
||||||
@@ -47,16 +51,21 @@ class IndeedScraper(Scraper):
|
|||||||
:param session:
|
:param session:
|
||||||
:return: jobs found on page, total number of jobs found for search
|
:return: jobs found on page, total number of jobs found for search
|
||||||
"""
|
"""
|
||||||
|
self.country = scraper_input.country
|
||||||
|
domain = self.country.domain_value
|
||||||
|
self.url = f"https://{domain}.indeed.com"
|
||||||
|
|
||||||
job_list = []
|
job_list: list[JobPost] = []
|
||||||
|
|
||||||
params = {
|
params = {
|
||||||
"q": scraper_input.search_term,
|
"q": scraper_input.search_term,
|
||||||
"l": scraper_input.location,
|
"l": scraper_input.location,
|
||||||
"radius": scraper_input.distance,
|
|
||||||
"filter": 0,
|
"filter": 0,
|
||||||
"start": 0 + page * 10,
|
"start": 0 + page * 10,
|
||||||
}
|
}
|
||||||
|
if scraper_input.distance:
|
||||||
|
params["radius"] = scraper_input.distance
|
||||||
|
|
||||||
sc_values = []
|
sc_values = []
|
||||||
if scraper_input.is_remote:
|
if scraper_input.is_remote:
|
||||||
sc_values.append("attr(DSQF7)")
|
sc_values.append("attr(DSQF7)")
|
||||||
@@ -65,14 +74,26 @@ class IndeedScraper(Scraper):
|
|||||||
|
|
||||||
if sc_values:
|
if sc_values:
|
||||||
params["sc"] = "0kf:" + "".join(sc_values) + ";"
|
params["sc"] = "0kf:" + "".join(sc_values) + ";"
|
||||||
response = session.get(self.url + "/jobs", params=params)
|
try:
|
||||||
|
response = session.get(
|
||||||
if response.status_code != 200 and response.status_code != 307:
|
self.url + "/jobs",
|
||||||
raise StatusException(response.status_code)
|
params=params,
|
||||||
|
allow_redirects=True,
|
||||||
|
proxy=self.proxy,
|
||||||
|
timeout_seconds=10,
|
||||||
|
)
|
||||||
|
if response.status_code not in range(200, 400):
|
||||||
|
raise IndeedException(
|
||||||
|
f"bad response with status code: {response.status_code}"
|
||||||
|
)
|
||||||
|
except Exception as e:
|
||||||
|
if "Proxy responded with" in str(e):
|
||||||
|
raise IndeedException("bad proxy")
|
||||||
|
raise IndeedException(str(e))
|
||||||
|
|
||||||
soup = BeautifulSoup(response.content, "html.parser")
|
soup = BeautifulSoup(response.content, "html.parser")
|
||||||
if "did not match any jobs" in str(soup):
|
if "did not match any jobs" in response.text:
|
||||||
raise ParsingException("Search did not match any jobs")
|
raise IndeedException("Parsing exception: Search did not match any jobs")
|
||||||
|
|
||||||
jobs = IndeedScraper.parse_jobs(
|
jobs = IndeedScraper.parse_jobs(
|
||||||
soup
|
soup
|
||||||
@@ -84,7 +105,7 @@ class IndeedScraper(Scraper):
|
|||||||
.get("mosaicProviderJobCardsModel", {})
|
.get("mosaicProviderJobCardsModel", {})
|
||||||
.get("results")
|
.get("results")
|
||||||
):
|
):
|
||||||
raise Exception("No jobs found.")
|
raise IndeedException("No jobs found.")
|
||||||
|
|
||||||
def process_job(job) -> Optional[JobPost]:
|
def process_job(job) -> Optional[JobPost]:
|
||||||
job_url = f'{self.url}/jobs/viewjob?jk={job["jobkey"]}'
|
job_url = f'{self.url}/jobs/viewjob?jk={job["jobkey"]}'
|
||||||
@@ -92,8 +113,6 @@ class IndeedScraper(Scraper):
|
|||||||
if job_url in self.seen_urls:
|
if job_url in self.seen_urls:
|
||||||
return None
|
return None
|
||||||
|
|
||||||
snippet_html = BeautifulSoup(job["snippet"], "html.parser")
|
|
||||||
|
|
||||||
extracted_salary = job.get("extractedSalary")
|
extracted_salary = job.get("extractedSalary")
|
||||||
compensation = None
|
compensation = None
|
||||||
if extracted_salary:
|
if extracted_salary:
|
||||||
@@ -118,11 +137,12 @@ class IndeedScraper(Scraper):
|
|||||||
date_posted = date_posted.strftime("%Y-%m-%d")
|
date_posted = date_posted.strftime("%Y-%m-%d")
|
||||||
|
|
||||||
description = self.get_description(job_url, session)
|
description = self.get_description(job_url, session)
|
||||||
li_elements = snippet_html.find_all("li")
|
with io.StringIO(job["snippet"]) as f:
|
||||||
if description is None and li_elements:
|
soup = BeautifulSoup(f, "html.parser")
|
||||||
description = " ".join(li.text for li in li_elements)
|
li_elements = soup.find_all("li")
|
||||||
|
if description is None and li_elements:
|
||||||
|
description = " ".join(li.text for li in li_elements)
|
||||||
|
|
||||||
first_li = snippet_html.find("li")
|
|
||||||
job_post = JobPost(
|
job_post = JobPost(
|
||||||
title=job["normTitle"],
|
title=job["normTitle"],
|
||||||
description=description,
|
description=description,
|
||||||
@@ -130,6 +150,7 @@ class IndeedScraper(Scraper):
|
|||||||
location=Location(
|
location=Location(
|
||||||
city=job.get("jobLocationCity"),
|
city=job.get("jobLocationCity"),
|
||||||
state=job.get("jobLocationState"),
|
state=job.get("jobLocationState"),
|
||||||
|
country=self.country,
|
||||||
),
|
),
|
||||||
job_type=job_type,
|
job_type=job_type,
|
||||||
compensation=compensation,
|
compensation=compensation,
|
||||||
@@ -138,7 +159,7 @@ class IndeedScraper(Scraper):
|
|||||||
)
|
)
|
||||||
return job_post
|
return job_post
|
||||||
|
|
||||||
with ThreadPoolExecutor(max_workers=10) as executor:
|
with ThreadPoolExecutor(max_workers=1) as executor:
|
||||||
job_results: list[Future] = [
|
job_results: list[Future] = [
|
||||||
executor.submit(process_job, job)
|
executor.submit(process_job, job)
|
||||||
for job in jobs["metaData"]["mosaicProviderJobCardsModel"]["results"]
|
for job in jobs["metaData"]["mosaicProviderJobCardsModel"]["results"]
|
||||||
@@ -162,42 +183,24 @@ class IndeedScraper(Scraper):
|
|||||||
math.ceil(scraper_input.results_wanted / self.jobs_per_page) - 1
|
math.ceil(scraper_input.results_wanted / self.jobs_per_page) - 1
|
||||||
)
|
)
|
||||||
|
|
||||||
try:
|
#: get first page to initialize session
|
||||||
#: get first page to initialize session
|
job_list, total_results = self.scrape_page(scraper_input, 0, session)
|
||||||
job_list, total_results = self.scrape_page(scraper_input, 0, session)
|
|
||||||
|
|
||||||
with ThreadPoolExecutor(max_workers=10) as executor:
|
with ThreadPoolExecutor(max_workers=1) as executor:
|
||||||
futures: list[Future] = [
|
futures: list[Future] = [
|
||||||
executor.submit(self.scrape_page, scraper_input, page, session)
|
executor.submit(self.scrape_page, scraper_input, page, session)
|
||||||
for page in range(1, pages_to_process + 1)
|
for page in range(1, pages_to_process + 1)
|
||||||
]
|
]
|
||||||
|
|
||||||
for future in futures:
|
for future in futures:
|
||||||
jobs, _ = future.result()
|
jobs, _ = future.result()
|
||||||
|
|
||||||
job_list += jobs
|
job_list += jobs
|
||||||
except StatusException as e:
|
|
||||||
return JobResponse(
|
|
||||||
success=False,
|
|
||||||
error=f"Indeed returned status code {e.status_code}",
|
|
||||||
)
|
|
||||||
|
|
||||||
except ParsingException as e:
|
|
||||||
return JobResponse(
|
|
||||||
success=False,
|
|
||||||
error=f"Indeed failed to parse response: {e}",
|
|
||||||
)
|
|
||||||
except Exception as e:
|
|
||||||
return JobResponse(
|
|
||||||
success=False,
|
|
||||||
error=f"Indeed failed to scrape: {e}",
|
|
||||||
)
|
|
||||||
|
|
||||||
if len(job_list) > scraper_input.results_wanted:
|
if len(job_list) > scraper_input.results_wanted:
|
||||||
job_list = job_list[: scraper_input.results_wanted]
|
job_list = job_list[: scraper_input.results_wanted]
|
||||||
|
|
||||||
job_response = JobResponse(
|
job_response = JobResponse(
|
||||||
success=True,
|
|
||||||
jobs=job_list,
|
jobs=job_list,
|
||||||
total_results=total_results,
|
total_results=total_results,
|
||||||
)
|
)
|
||||||
@@ -215,7 +218,12 @@ class IndeedScraper(Scraper):
|
|||||||
jk_value = params.get("jk", [None])[0]
|
jk_value = params.get("jk", [None])[0]
|
||||||
formatted_url = f"{self.url}/viewjob?jk={jk_value}&spa=1"
|
formatted_url = f"{self.url}/viewjob?jk={jk_value}&spa=1"
|
||||||
|
|
||||||
response = session.get(formatted_url, allow_redirects=True)
|
try:
|
||||||
|
response = session.get(
|
||||||
|
formatted_url, allow_redirects=True, timeout_seconds=5, proxy=self.proxy
|
||||||
|
)
|
||||||
|
except Exception as e:
|
||||||
|
return None
|
||||||
|
|
||||||
if response.status_code not in range(200, 400):
|
if response.status_code not in range(200, 400):
|
||||||
return None
|
return None
|
||||||
@@ -223,9 +231,10 @@ class IndeedScraper(Scraper):
|
|||||||
raw_description = response.json()["body"]["jobInfoWrapperModel"][
|
raw_description = response.json()["body"]["jobInfoWrapperModel"][
|
||||||
"jobInfoModel"
|
"jobInfoModel"
|
||||||
]["sanitizedJobDescription"]
|
]["sanitizedJobDescription"]
|
||||||
soup = BeautifulSoup(raw_description, "html.parser")
|
with io.StringIO(raw_description) as f:
|
||||||
text_content = " ".join(soup.get_text().split()).strip()
|
soup = BeautifulSoup(f, "html.parser")
|
||||||
return text_content
|
text_content = " ".join(soup.get_text().split()).strip()
|
||||||
|
return text_content
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def get_job_type(job: dict) -> Optional[JobType]:
|
def get_job_type(job: dict) -> Optional[JobType]:
|
||||||
@@ -237,13 +246,17 @@ class IndeedScraper(Scraper):
|
|||||||
for taxonomy in job["taxonomyAttributes"]:
|
for taxonomy in job["taxonomyAttributes"]:
|
||||||
if taxonomy["label"] == "job-types":
|
if taxonomy["label"] == "job-types":
|
||||||
if len(taxonomy["attributes"]) > 0:
|
if len(taxonomy["attributes"]) > 0:
|
||||||
job_type_str = (
|
label = taxonomy["attributes"][0].get("label")
|
||||||
taxonomy["attributes"][0]["label"]
|
if label:
|
||||||
.replace("-", "_")
|
job_type_str = label.replace("-", "").replace(" ", "").lower()
|
||||||
.replace(" ", "_")
|
return IndeedScraper.get_enum_from_value(job_type_str)
|
||||||
.upper()
|
return None
|
||||||
)
|
|
||||||
return JobType[job_type_str]
|
@staticmethod
|
||||||
|
def get_enum_from_value(value_str):
|
||||||
|
for job_type in JobType:
|
||||||
|
if value_str in job_type.value:
|
||||||
|
return job_type
|
||||||
return None
|
return None
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
@@ -281,9 +294,9 @@ class IndeedScraper(Scraper):
|
|||||||
jobs = json.loads(m.group(1).strip())
|
jobs = json.loads(m.group(1).strip())
|
||||||
return jobs
|
return jobs
|
||||||
else:
|
else:
|
||||||
raise ParsingException("Could not find mosaic provider job cards data")
|
raise IndeedException("Could not find mosaic provider job cards data")
|
||||||
else:
|
else:
|
||||||
raise ParsingException(
|
raise IndeedException(
|
||||||
"Could not find a script tag containing mosaic provider data"
|
"Could not find a script tag containing mosaic provider data"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -294,7 +307,7 @@ class IndeedScraper(Scraper):
|
|||||||
:param soup:
|
:param soup:
|
||||||
:return: total_num_jobs
|
:return: total_num_jobs
|
||||||
"""
|
"""
|
||||||
script = soup.find("script", string=lambda t: "window._initialData" in t)
|
script = soup.find("script", string=lambda t: t and "window._initialData" in t)
|
||||||
|
|
||||||
pattern = re.compile(r"window._initialData\s*=\s*({.*})\s*;", re.DOTALL)
|
pattern = re.compile(r"window._initialData\s*=\s*({.*})\s*;", re.DOTALL)
|
||||||
match = pattern.search(script.string)
|
match = pattern.search(script.string)
|
||||||
|
|||||||
@@ -1,11 +1,20 @@
|
|||||||
|
"""
|
||||||
|
jobspy.scrapers.linkedin
|
||||||
|
~~~~~~~~~~~~~~~~~~~
|
||||||
|
|
||||||
|
This module contains routines to scrape LinkedIn.
|
||||||
|
"""
|
||||||
from typing import Optional, Tuple
|
from typing import Optional, Tuple
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
|
import traceback
|
||||||
|
|
||||||
import requests
|
import requests
|
||||||
|
from requests.exceptions import Timeout, ProxyError
|
||||||
from bs4 import BeautifulSoup
|
from bs4 import BeautifulSoup
|
||||||
from bs4.element import Tag
|
from bs4.element import Tag
|
||||||
|
|
||||||
from .. import Scraper, ScraperInput, Site
|
from .. import Scraper, ScraperInput, Site
|
||||||
|
from ..exceptions import LinkedInException
|
||||||
from ...jobs import (
|
from ...jobs import (
|
||||||
JobPost,
|
JobPost,
|
||||||
Location,
|
Location,
|
||||||
@@ -17,13 +26,13 @@ from ...jobs import (
|
|||||||
|
|
||||||
|
|
||||||
class LinkedInScraper(Scraper):
|
class LinkedInScraper(Scraper):
|
||||||
def __init__(self):
|
def __init__(self, proxy: Optional[str] = None):
|
||||||
"""
|
"""
|
||||||
Initializes LinkedInScraper with the LinkedIn job search url
|
Initializes LinkedInScraper with the LinkedIn job search url
|
||||||
"""
|
"""
|
||||||
site = Site(Site.LINKEDIN)
|
site = Site(Site.LINKEDIN)
|
||||||
url = "https://www.linkedin.com"
|
self.url = "https://www.linkedin.com"
|
||||||
super().__init__(site, url)
|
super().__init__(site, proxy=proxy)
|
||||||
|
|
||||||
def scrape(self, scraper_input: ScraperInput) -> JobResponse:
|
def scrape(self, scraper_input: ScraperInput) -> JobResponse:
|
||||||
"""
|
"""
|
||||||
@@ -31,6 +40,7 @@ class LinkedInScraper(Scraper):
|
|||||||
:param scraper_input:
|
:param scraper_input:
|
||||||
:return: job_response
|
:return: job_response
|
||||||
"""
|
"""
|
||||||
|
self.country = "worldwide"
|
||||||
job_list: list[JobPost] = []
|
job_list: list[JobPost] = []
|
||||||
seen_urls = set()
|
seen_urls = set()
|
||||||
page, processed_jobs, job_count = 0, 0, 0
|
page, processed_jobs, job_count = 0, 0, 0
|
||||||
@@ -61,15 +71,23 @@ class LinkedInScraper(Scraper):
|
|||||||
}
|
}
|
||||||
|
|
||||||
params = {k: v for k, v in params.items() if v is not None}
|
params = {k: v for k, v in params.items() if v is not None}
|
||||||
response = session.get(
|
try:
|
||||||
f"{self.url}/jobs/search", params=params, allow_redirects=True
|
response = session.get(
|
||||||
)
|
f"{self.url}/jobs/search",
|
||||||
|
params=params,
|
||||||
if response.status_code != 200:
|
allow_redirects=True,
|
||||||
return JobResponse(
|
proxies=self.proxy,
|
||||||
success=False,
|
timeout=10,
|
||||||
error=f"Response returned {response.status_code}",
|
|
||||||
)
|
)
|
||||||
|
response.raise_for_status()
|
||||||
|
except requests.HTTPError as e:
|
||||||
|
raise LinkedInException(
|
||||||
|
f"bad response status code: {response.status_code}"
|
||||||
|
)
|
||||||
|
except ProxyError as e:
|
||||||
|
raise LinkedInException("bad proxy")
|
||||||
|
except (ProxyError, Exception) as e:
|
||||||
|
raise LinkedInException(str(e))
|
||||||
|
|
||||||
soup = BeautifulSoup(response.text, "html.parser")
|
soup = BeautifulSoup(response.text, "html.parser")
|
||||||
|
|
||||||
@@ -104,15 +122,18 @@ class LinkedInScraper(Scraper):
|
|||||||
metadata_card = job_info.find(
|
metadata_card = job_info.find(
|
||||||
"div", class_="base-search-card__metadata"
|
"div", class_="base-search-card__metadata"
|
||||||
)
|
)
|
||||||
location: Location = LinkedInScraper.get_location(metadata_card)
|
location: Location = self.get_location(metadata_card)
|
||||||
|
|
||||||
datetime_tag = metadata_card.find(
|
datetime_tag = metadata_card.find(
|
||||||
"time", class_="job-search-card__listdate"
|
"time", class_="job-search-card__listdate"
|
||||||
)
|
)
|
||||||
description, job_type = LinkedInScraper.get_description(job_url)
|
description, job_type = self.get_description(job_url)
|
||||||
if datetime_tag:
|
if datetime_tag:
|
||||||
datetime_str = datetime_tag["datetime"]
|
datetime_str = datetime_tag["datetime"]
|
||||||
date_posted = datetime.strptime(datetime_str, "%Y-%m-%d")
|
try:
|
||||||
|
date_posted = datetime.strptime(datetime_str, "%Y-%m-%d")
|
||||||
|
except Exception as e:
|
||||||
|
date_posted = None
|
||||||
else:
|
else:
|
||||||
date_posted = None
|
date_posted = None
|
||||||
|
|
||||||
@@ -125,40 +146,34 @@ class LinkedInScraper(Scraper):
|
|||||||
job_url=job_url,
|
job_url=job_url,
|
||||||
job_type=job_type,
|
job_type=job_type,
|
||||||
compensation=Compensation(
|
compensation=Compensation(
|
||||||
interval=CompensationInterval.YEARLY, currency="USD"
|
interval=CompensationInterval.YEARLY, currency=None
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
job_list.append(job_post)
|
job_list.append(job_post)
|
||||||
if (
|
if processed_jobs >= job_count:
|
||||||
len(job_list) >= scraper_input.results_wanted
|
|
||||||
or processed_jobs >= job_count
|
|
||||||
):
|
|
||||||
break
|
break
|
||||||
if (
|
if len(job_list) >= scraper_input.results_wanted:
|
||||||
len(job_list) >= scraper_input.results_wanted
|
break
|
||||||
or processed_jobs >= job_count
|
if processed_jobs >= job_count:
|
||||||
):
|
break
|
||||||
|
if len(job_list) >= scraper_input.results_wanted:
|
||||||
break
|
break
|
||||||
|
|
||||||
page += 1
|
page += 1
|
||||||
|
|
||||||
job_list = job_list[: scraper_input.results_wanted]
|
job_list = job_list[: scraper_input.results_wanted]
|
||||||
job_response = JobResponse(
|
return JobResponse(jobs=job_list)
|
||||||
success=True,
|
|
||||||
jobs=job_list,
|
|
||||||
total_results=job_count,
|
|
||||||
)
|
|
||||||
return job_response
|
|
||||||
|
|
||||||
@staticmethod
|
def get_description(self, job_page_url: str) -> Optional[str]:
|
||||||
def get_description(job_page_url: str) -> Optional[str]:
|
|
||||||
"""
|
"""
|
||||||
Retrieves job description by going to the job page url
|
Retrieves job description by going to the job page url
|
||||||
:param job_page_url:
|
:param job_page_url:
|
||||||
:return: description or None
|
:return: description or None
|
||||||
"""
|
"""
|
||||||
response = requests.get(job_page_url, allow_redirects=True)
|
try:
|
||||||
if response.status_code not in range(200, 400):
|
response = requests.get(job_page_url, timeout=5, proxies=self.proxy)
|
||||||
|
response.raise_for_status()
|
||||||
|
except Exception as e:
|
||||||
return None, None
|
return None, None
|
||||||
|
|
||||||
soup = BeautifulSoup(response.text, "html.parser")
|
soup = BeautifulSoup(response.text, "html.parser")
|
||||||
@@ -195,17 +210,24 @@ class LinkedInScraper(Scraper):
|
|||||||
employment_type = employment_type.lower()
|
employment_type = employment_type.lower()
|
||||||
employment_type = employment_type.replace("-", "")
|
employment_type = employment_type.replace("-", "")
|
||||||
|
|
||||||
return JobType(employment_type)
|
return LinkedInScraper.get_enum_from_value(employment_type)
|
||||||
|
|
||||||
return text_content, get_job_type(soup)
|
return text_content, get_job_type(soup)
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def get_location(metadata_card: Optional[Tag]) -> Location:
|
def get_enum_from_value(value_str):
|
||||||
|
for job_type in JobType:
|
||||||
|
if value_str in job_type.value:
|
||||||
|
return job_type
|
||||||
|
return None
|
||||||
|
|
||||||
|
def get_location(self, metadata_card: Optional[Tag]) -> Location:
|
||||||
"""
|
"""
|
||||||
Extracts the location data from the job metadata card.
|
Extracts the location data from the job metadata card.
|
||||||
:param metadata_card
|
:param metadata_card
|
||||||
:return: location
|
:return: location
|
||||||
"""
|
"""
|
||||||
|
location = Location(country=self.country)
|
||||||
if metadata_card is not None:
|
if metadata_card is not None:
|
||||||
location_tag = metadata_card.find(
|
location_tag = metadata_card.find(
|
||||||
"span", class_="job-search-card__location"
|
"span", class_="job-search-card__location"
|
||||||
@@ -217,6 +239,7 @@ class LinkedInScraper(Scraper):
|
|||||||
location = Location(
|
location = Location(
|
||||||
city=city,
|
city=city,
|
||||||
state=state,
|
state=state,
|
||||||
|
country=self.country,
|
||||||
)
|
)
|
||||||
|
|
||||||
return location
|
return location
|
||||||
|
|||||||
@@ -1,16 +1,25 @@
|
|||||||
|
"""
|
||||||
|
jobspy.scrapers.ziprecruiter
|
||||||
|
~~~~~~~~~~~~~~~~~~~
|
||||||
|
|
||||||
|
This module contains routines to scrape ZipRecruiter.
|
||||||
|
"""
|
||||||
import math
|
import math
|
||||||
import json
|
import json
|
||||||
import re
|
import re
|
||||||
|
import traceback
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
from typing import Optional, Tuple
|
from typing import Optional, Tuple
|
||||||
from urllib.parse import urlparse, parse_qs
|
from urllib.parse import urlparse, parse_qs
|
||||||
|
|
||||||
import tls_client
|
import tls_client
|
||||||
|
import requests
|
||||||
from bs4 import BeautifulSoup
|
from bs4 import BeautifulSoup
|
||||||
from bs4.element import Tag
|
from bs4.element import Tag
|
||||||
from concurrent.futures import ThreadPoolExecutor, Future
|
from concurrent.futures import ThreadPoolExecutor, Future
|
||||||
|
|
||||||
from .. import Scraper, ScraperInput, Site, StatusException
|
from .. import Scraper, ScraperInput, Site
|
||||||
|
from ..exceptions import ZipRecruiterException
|
||||||
from ...jobs import (
|
from ...jobs import (
|
||||||
JobPost,
|
JobPost,
|
||||||
Compensation,
|
Compensation,
|
||||||
@@ -18,17 +27,18 @@ from ...jobs import (
|
|||||||
Location,
|
Location,
|
||||||
JobResponse,
|
JobResponse,
|
||||||
JobType,
|
JobType,
|
||||||
|
Country,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
class ZipRecruiterScraper(Scraper):
|
class ZipRecruiterScraper(Scraper):
|
||||||
def __init__(self):
|
def __init__(self, proxy: Optional[str] = None):
|
||||||
"""
|
"""
|
||||||
Initializes LinkedInScraper with the ZipRecruiter job search url
|
Initializes LinkedInScraper with the ZipRecruiter job search url
|
||||||
"""
|
"""
|
||||||
site = Site(Site.ZIP_RECRUITER)
|
site = Site(Site.ZIP_RECRUITER)
|
||||||
url = "https://www.ziprecruiter.com"
|
self.url = "https://www.ziprecruiter.com"
|
||||||
super().__init__(site, url)
|
super().__init__(site, proxy=proxy)
|
||||||
|
|
||||||
self.jobs_per_page = 20
|
self.jobs_per_page = 20
|
||||||
self.seen_urls = set()
|
self.seen_urls = set()
|
||||||
@@ -36,7 +46,7 @@ class ZipRecruiterScraper(Scraper):
|
|||||||
client_identifier="chrome112", random_tls_extension_order=True
|
client_identifier="chrome112", random_tls_extension_order=True
|
||||||
)
|
)
|
||||||
|
|
||||||
def scrape_page(
|
def find_jobs_in_page(
|
||||||
self, scraper_input: ScraperInput, page: int
|
self, scraper_input: ScraperInput, page: int
|
||||||
) -> tuple[list[JobPost], int | None]:
|
) -> tuple[list[JobPost], int | None]:
|
||||||
"""
|
"""
|
||||||
@@ -46,71 +56,62 @@ class ZipRecruiterScraper(Scraper):
|
|||||||
:param session:
|
:param session:
|
||||||
:return: jobs found on page, total number of jobs found for search
|
:return: jobs found on page, total number of jobs found for search
|
||||||
"""
|
"""
|
||||||
|
job_list: list[JobPost] = []
|
||||||
job_list = []
|
try:
|
||||||
|
response = self.session.get(
|
||||||
job_type_value = None
|
self.url + "/jobs-search",
|
||||||
if scraper_input.job_type:
|
headers=ZipRecruiterScraper.headers(),
|
||||||
if scraper_input.job_type.value == "fulltime":
|
params=ZipRecruiterScraper.add_params(scraper_input, page),
|
||||||
job_type_value = "full_time"
|
allow_redirects=True,
|
||||||
elif scraper_input.job_type.value == "parttime":
|
proxy=self.proxy,
|
||||||
job_type_value = "part_time"
|
timeout_seconds=10,
|
||||||
else:
|
)
|
||||||
job_type_value = scraper_input.job_type.value
|
if response.status_code != 200:
|
||||||
|
raise ZipRecruiterException(
|
||||||
params = {
|
f"bad response status code: {response.status_code}"
|
||||||
"search": scraper_input.search_term,
|
)
|
||||||
"location": scraper_input.location,
|
except Exception as e:
|
||||||
"page": page,
|
if "Proxy responded with non 200 code" in str(e):
|
||||||
"form": "jobs-landing",
|
raise ZipRecruiterException("bad proxy")
|
||||||
}
|
raise ZipRecruiterException(str(e))
|
||||||
|
|
||||||
if scraper_input.is_remote:
|
|
||||||
params["refine_by_location_type"] = "only_remote"
|
|
||||||
|
|
||||||
if scraper_input.distance:
|
|
||||||
params["radius"] = scraper_input.distance
|
|
||||||
|
|
||||||
if job_type_value:
|
|
||||||
params[
|
|
||||||
"refine_by_employment"
|
|
||||||
] = f"employment_type:employment_type:{job_type_value}"
|
|
||||||
|
|
||||||
response = self.session.get(
|
|
||||||
self.url + "/jobs-search",
|
|
||||||
headers=ZipRecruiterScraper.headers(),
|
|
||||||
params=params,
|
|
||||||
)
|
|
||||||
|
|
||||||
if response.status_code != 200:
|
|
||||||
raise StatusException(response.status_code)
|
|
||||||
|
|
||||||
html_string = response.text
|
|
||||||
soup = BeautifulSoup(html_string, "html.parser")
|
|
||||||
|
|
||||||
script_tag = soup.find("script", {"id": "js_variables"})
|
|
||||||
data = json.loads(script_tag.string)
|
|
||||||
|
|
||||||
if page == 1:
|
|
||||||
job_count = int(data["totalJobCount"].replace(",", ""))
|
|
||||||
else:
|
else:
|
||||||
job_count = None
|
soup = BeautifulSoup(response.text, "html.parser")
|
||||||
|
js_tag = soup.find("script", {"id": "js_variables"})
|
||||||
|
|
||||||
|
if js_tag:
|
||||||
|
page_json = json.loads(js_tag.string)
|
||||||
|
jobs_list = page_json.get("jobList")
|
||||||
|
if jobs_list:
|
||||||
|
page_variant = "javascript"
|
||||||
|
# print('type javascript', len(jobs_list))
|
||||||
|
else:
|
||||||
|
page_variant = "html_2"
|
||||||
|
jobs_list = soup.find_all("div", {"class": "job_content"})
|
||||||
|
# print('type 2 html', len(jobs_list))
|
||||||
|
else:
|
||||||
|
page_variant = "html_1"
|
||||||
|
jobs_list = soup.find_all("li", {"class": "job-listing"})
|
||||||
|
# print('type 1 html', len(jobs_list))
|
||||||
|
# with open("zip_method_8.html", "w") as f:
|
||||||
|
# f.write(soup.prettify())
|
||||||
|
|
||||||
with ThreadPoolExecutor(max_workers=10) as executor:
|
with ThreadPoolExecutor(max_workers=10) as executor:
|
||||||
if "jobList" in data and data["jobList"]:
|
if page_variant == "javascript":
|
||||||
jobs_js = data["jobList"]
|
|
||||||
job_results = [
|
job_results = [
|
||||||
executor.submit(self.process_job_js, job) for job in jobs_js
|
executor.submit(self.process_job_javascript, job)
|
||||||
|
for job in jobs_list
|
||||||
]
|
]
|
||||||
else:
|
elif page_variant == "html_1":
|
||||||
jobs_html = soup.find_all("div", {"class": "job_content"})
|
|
||||||
job_results = [
|
job_results = [
|
||||||
executor.submit(self.process_job_html, job) for job in jobs_html
|
executor.submit(self.process_job_html_1, job) for job in jobs_list
|
||||||
|
]
|
||||||
|
elif page_variant == "html_2":
|
||||||
|
job_results = [
|
||||||
|
executor.submit(self.process_job_html_2, job) for job in jobs_list
|
||||||
]
|
]
|
||||||
|
|
||||||
job_list = [result.result() for result in job_results if result.result()]
|
job_list = [result.result() for result in job_results if result.result()]
|
||||||
|
return job_list
|
||||||
return job_list, job_count
|
|
||||||
|
|
||||||
def scrape(self, scraper_input: ScraperInput) -> JobResponse:
|
def scrape(self, scraper_input: ScraperInput) -> JobResponse:
|
||||||
"""
|
"""
|
||||||
@@ -118,50 +119,27 @@ class ZipRecruiterScraper(Scraper):
|
|||||||
:param scraper_input:
|
:param scraper_input:
|
||||||
:return: job_response
|
:return: job_response
|
||||||
"""
|
"""
|
||||||
|
#: get first page to initialize session
|
||||||
|
job_list: list[JobPost] = self.find_jobs_in_page(scraper_input, 1)
|
||||||
pages_to_process = max(
|
pages_to_process = max(
|
||||||
3, math.ceil(scraper_input.results_wanted / self.jobs_per_page)
|
3, math.ceil(scraper_input.results_wanted / self.jobs_per_page)
|
||||||
)
|
)
|
||||||
|
|
||||||
try:
|
with ThreadPoolExecutor(max_workers=10) as executor:
|
||||||
#: get first page to initialize session
|
futures: list[Future] = [
|
||||||
job_list, total_results = self.scrape_page(scraper_input, 1)
|
executor.submit(self.find_jobs_in_page, scraper_input, page)
|
||||||
|
for page in range(2, pages_to_process + 1)
|
||||||
|
]
|
||||||
|
|
||||||
with ThreadPoolExecutor(max_workers=10) as executor:
|
for future in futures:
|
||||||
futures: list[Future] = [
|
jobs = future.result()
|
||||||
executor.submit(self.scrape_page, scraper_input, page)
|
|
||||||
for page in range(2, pages_to_process + 1)
|
|
||||||
]
|
|
||||||
|
|
||||||
for future in futures:
|
job_list += jobs
|
||||||
jobs, _ = future.result()
|
|
||||||
|
|
||||||
job_list += jobs
|
job_list = job_list[: scraper_input.results_wanted]
|
||||||
|
return JobResponse(jobs=job_list)
|
||||||
|
|
||||||
except StatusException as e:
|
def process_job_html_1(self, job: Tag) -> Optional[JobPost]:
|
||||||
return JobResponse(
|
|
||||||
success=False,
|
|
||||||
error=f"ZipRecruiter returned status code {e.status_code}",
|
|
||||||
)
|
|
||||||
except Exception as e:
|
|
||||||
return JobResponse(
|
|
||||||
success=False,
|
|
||||||
error=f"ZipRecruiter failed to scrape: {e}",
|
|
||||||
)
|
|
||||||
|
|
||||||
#: note: this does not handle if the results are more or less than the results_wanted
|
|
||||||
|
|
||||||
if len(job_list) > scraper_input.results_wanted:
|
|
||||||
job_list = job_list[: scraper_input.results_wanted]
|
|
||||||
|
|
||||||
job_response = JobResponse(
|
|
||||||
success=True,
|
|
||||||
jobs=job_list,
|
|
||||||
total_results=total_results,
|
|
||||||
)
|
|
||||||
return job_response
|
|
||||||
|
|
||||||
def process_job_html(self, job: Tag) -> Optional[JobPost]:
|
|
||||||
"""
|
"""
|
||||||
Parses a job from the job content tag
|
Parses a job from the job content tag
|
||||||
:param job: BeautifulSoup Tag for one job post
|
:param job: BeautifulSoup Tag for one job post
|
||||||
@@ -175,21 +153,17 @@ class ZipRecruiterScraper(Scraper):
|
|||||||
company = job.find("a", {"class": "company_name"}).text.strip()
|
company = job.find("a", {"class": "company_name"}).text.strip()
|
||||||
|
|
||||||
description, updated_job_url = self.get_description(job_url)
|
description, updated_job_url = self.get_description(job_url)
|
||||||
if updated_job_url is not None:
|
job_url = updated_job_url if updated_job_url else job_url
|
||||||
job_url = updated_job_url
|
|
||||||
if description is None:
|
if description is None:
|
||||||
description = job.find("p", {"class": "job_snippet"}).text.strip()
|
description = job.find("p", {"class": "job_snippet"}).text.strip()
|
||||||
|
|
||||||
job_type_element = job.find("li", {"class": "perk_item perk_type"})
|
job_type_element = job.find("li", {"class": "perk_item perk_type"})
|
||||||
|
job_type = None
|
||||||
if job_type_element:
|
if job_type_element:
|
||||||
job_type_text = (
|
job_type_text = (
|
||||||
job_type_element.text.strip().lower().replace("-", "").replace(" ", "")
|
job_type_element.text.strip().lower().replace("_", "").replace(" ", "")
|
||||||
)
|
)
|
||||||
if job_type_text == "contractor":
|
job_type = ZipRecruiterScraper.get_job_type_enum(job_type_text)
|
||||||
job_type_text = "contract"
|
|
||||||
job_type = JobType(job_type_text)
|
|
||||||
else:
|
|
||||||
job_type = None
|
|
||||||
|
|
||||||
date_posted = ZipRecruiterScraper.get_date_posted(job)
|
date_posted = ZipRecruiterScraper.get_date_posted(job)
|
||||||
|
|
||||||
@@ -205,22 +179,64 @@ class ZipRecruiterScraper(Scraper):
|
|||||||
)
|
)
|
||||||
return job_post
|
return job_post
|
||||||
|
|
||||||
def process_job_js(self, job: dict) -> JobPost:
|
def process_job_html_2(self, job: Tag) -> Optional[JobPost]:
|
||||||
# Map the job data to the expected fields by the Pydantic model
|
"""
|
||||||
|
Parses a job from the job content tag for a second variat of HTML that ZR uses
|
||||||
|
:param job: BeautifulSoup Tag for one job post
|
||||||
|
:return JobPost
|
||||||
|
"""
|
||||||
|
job_url = job.find("a", class_="job_link")["href"]
|
||||||
|
title = job.find("h2", class_="title").text
|
||||||
|
company = job.find("a", class_="company_name").text.strip()
|
||||||
|
|
||||||
|
description, updated_job_url = self.get_description(job_url)
|
||||||
|
job_url = updated_job_url if updated_job_url else job_url
|
||||||
|
if description is None:
|
||||||
|
description = job.find("p", class_="job_snippet").get_text().strip()
|
||||||
|
|
||||||
|
job_type_text = job.find("li", class_="perk_item perk_type")
|
||||||
|
job_type = None
|
||||||
|
if job_type_text:
|
||||||
|
job_type_text = (
|
||||||
|
job_type_text.get_text()
|
||||||
|
.strip()
|
||||||
|
.lower()
|
||||||
|
.replace("-", "")
|
||||||
|
.replace(" ", "")
|
||||||
|
)
|
||||||
|
job_type = ZipRecruiterScraper.get_job_type_enum(job_type_text)
|
||||||
|
date_posted = ZipRecruiterScraper.get_date_posted(job)
|
||||||
|
|
||||||
|
job_post = JobPost(
|
||||||
|
title=title,
|
||||||
|
description=description,
|
||||||
|
company_name=company,
|
||||||
|
location=ZipRecruiterScraper.get_location(job),
|
||||||
|
job_type=job_type,
|
||||||
|
compensation=ZipRecruiterScraper.get_compensation(job),
|
||||||
|
date_posted=date_posted,
|
||||||
|
job_url=job_url,
|
||||||
|
)
|
||||||
|
return job_post
|
||||||
|
|
||||||
|
def process_job_javascript(self, job: dict) -> JobPost:
|
||||||
title = job.get("Title")
|
title = job.get("Title")
|
||||||
description = BeautifulSoup(
|
job_url = job.get("JobURL")
|
||||||
job.get("Snippet", "").strip(), "html.parser"
|
|
||||||
).get_text()
|
description, updated_job_url = self.get_description(job_url)
|
||||||
|
job_url = updated_job_url if updated_job_url else job_url
|
||||||
|
if description is None:
|
||||||
|
description = BeautifulSoup(
|
||||||
|
job.get("Snippet", "").strip(), "html.parser"
|
||||||
|
).get_text()
|
||||||
|
|
||||||
company = job.get("OrgName")
|
company = job.get("OrgName")
|
||||||
location = Location(city=job.get("City"), state=job.get("State"))
|
location = Location(
|
||||||
try:
|
city=job.get("City"), state=job.get("State"), country=Country.US_CANADA
|
||||||
job_type = ZipRecruiterScraper.job_type_from_string(
|
)
|
||||||
job.get("EmploymentType", "").replace("-", "_").lower()
|
job_type = ZipRecruiterScraper.get_job_type_enum(
|
||||||
)
|
job.get("EmploymentType", "").replace("-", "").lower()
|
||||||
except ValueError:
|
)
|
||||||
# print(f"Skipping job due to unrecognized job type: {job.get('EmploymentType')}")
|
|
||||||
return None
|
|
||||||
|
|
||||||
formatted_salary = job.get("FormattedSalaryShort", "")
|
formatted_salary = job.get("FormattedSalaryShort", "")
|
||||||
salary_parts = formatted_salary.split(" ")
|
salary_parts = formatted_salary.split(" ")
|
||||||
@@ -244,6 +260,7 @@ class ZipRecruiterScraper(Scraper):
|
|||||||
interval=CompensationInterval.YEARLY,
|
interval=CompensationInterval.YEARLY,
|
||||||
min_amount=min_amount,
|
min_amount=min_amount,
|
||||||
max_amount=max_amount,
|
max_amount=max_amount,
|
||||||
|
currency="USD/CAD",
|
||||||
)
|
)
|
||||||
save_job_url = job.get("SaveJobURL", "")
|
save_job_url = job.get("SaveJobURL", "")
|
||||||
posted_time_match = re.search(
|
posted_time_match = re.search(
|
||||||
@@ -270,17 +287,12 @@ class ZipRecruiterScraper(Scraper):
|
|||||||
return job_post
|
return job_post
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def job_type_from_string(value: str) -> Optional[JobType]:
|
def get_job_type_enum(job_type_str: str) -> Optional[JobType]:
|
||||||
if not value:
|
for job_type in JobType:
|
||||||
return None
|
if job_type_str in job_type.value:
|
||||||
|
a = True
|
||||||
if value.lower() == "contractor":
|
return job_type
|
||||||
value = "contract"
|
return None
|
||||||
normalized_value = value.replace("_", "")
|
|
||||||
for item in JobType:
|
|
||||||
if item.value == normalized_value:
|
|
||||||
return item
|
|
||||||
raise ValueError(f"Invalid value for JobType: {value}")
|
|
||||||
|
|
||||||
def get_description(self, job_page_url: str) -> Tuple[Optional[str], Optional[str]]:
|
def get_description(self, job_page_url: str) -> Tuple[Optional[str], Optional[str]]:
|
||||||
"""
|
"""
|
||||||
@@ -289,10 +301,17 @@ class ZipRecruiterScraper(Scraper):
|
|||||||
:param session:
|
:param session:
|
||||||
:return: description or None, response url
|
:return: description or None, response url
|
||||||
"""
|
"""
|
||||||
response = self.session.get(
|
try:
|
||||||
job_page_url, headers=ZipRecruiterScraper.headers(), allow_redirects=True
|
response = requests.get(
|
||||||
)
|
job_page_url,
|
||||||
if response.status_code not in range(200, 400):
|
headers=ZipRecruiterScraper.headers(),
|
||||||
|
allow_redirects=True,
|
||||||
|
timeout=5,
|
||||||
|
proxies=self.proxy,
|
||||||
|
)
|
||||||
|
if response.status_code not in range(200, 400):
|
||||||
|
return None, None
|
||||||
|
except Exception as e:
|
||||||
return None, None
|
return None, None
|
||||||
|
|
||||||
html_string = response.content
|
html_string = response.content
|
||||||
@@ -303,6 +322,36 @@ class ZipRecruiterScraper(Scraper):
|
|||||||
return job_description_div.text.strip(), response.url
|
return job_description_div.text.strip(), response.url
|
||||||
return None, response.url
|
return None, response.url
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def add_params(scraper_input, page) -> Optional[str]:
|
||||||
|
params = {
|
||||||
|
"search": scraper_input.search_term,
|
||||||
|
"location": scraper_input.location,
|
||||||
|
"page": page,
|
||||||
|
"form": "jobs-landing",
|
||||||
|
}
|
||||||
|
job_type_value = None
|
||||||
|
if scraper_input.job_type:
|
||||||
|
if scraper_input.job_type.value == "fulltime":
|
||||||
|
job_type_value = "full_time"
|
||||||
|
elif scraper_input.job_type.value == "parttime":
|
||||||
|
job_type_value = "part_time"
|
||||||
|
else:
|
||||||
|
job_type_value = scraper_input.job_type.value
|
||||||
|
|
||||||
|
if job_type_value:
|
||||||
|
params[
|
||||||
|
"refine_by_employment"
|
||||||
|
] = f"employment_type:employment_type:{job_type_value}"
|
||||||
|
|
||||||
|
if scraper_input.is_remote:
|
||||||
|
params["refine_by_location_type"] = "only_remote"
|
||||||
|
|
||||||
|
if scraper_input.distance:
|
||||||
|
params["radius"] = scraper_input.distance
|
||||||
|
|
||||||
|
return params
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def get_interval(interval_str: str):
|
def get_interval(interval_str: str):
|
||||||
"""
|
"""
|
||||||
@@ -375,7 +424,10 @@ class ZipRecruiterScraper(Scraper):
|
|||||||
amounts.append(amount)
|
amounts.append(amount)
|
||||||
|
|
||||||
compensation = Compensation(
|
compensation = Compensation(
|
||||||
interval=interval, min_amount=min(amounts), max_amount=max(amounts)
|
interval=interval,
|
||||||
|
min_amount=min(amounts),
|
||||||
|
max_amount=max(amounts),
|
||||||
|
currency="USD/CAD",
|
||||||
)
|
)
|
||||||
|
|
||||||
return compensation
|
return compensation
|
||||||
@@ -399,10 +451,7 @@ class ZipRecruiterScraper(Scraper):
|
|||||||
city, state = None, None
|
city, state = None, None
|
||||||
else:
|
else:
|
||||||
city, state = None, None
|
city, state = None, None
|
||||||
return Location(
|
return Location(city=city, state=state, country=Country.US_CANADA)
|
||||||
city=city,
|
|
||||||
state=state,
|
|
||||||
)
|
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def headers() -> dict:
|
def headers() -> dict:
|
||||||
|
|||||||
10
src/tests/test_all.py
Normal file
10
src/tests/test_all.py
Normal file
@@ -0,0 +1,10 @@
|
|||||||
|
from ..jobspy import scrape_jobs
|
||||||
|
|
||||||
|
|
||||||
|
def test_all():
|
||||||
|
result = scrape_jobs(
|
||||||
|
site_name=["linkedin", "indeed", "zip_recruiter"],
|
||||||
|
search_term="software engineer",
|
||||||
|
results_wanted=5,
|
||||||
|
)
|
||||||
|
assert result is not None and result.errors.empty is True
|
||||||
@@ -6,4 +6,4 @@ def test_indeed():
|
|||||||
site_name="indeed",
|
site_name="indeed",
|
||||||
search_term="software engineer",
|
search_term="software engineer",
|
||||||
)
|
)
|
||||||
assert result is not None
|
assert result is not None and result.errors.empty is True
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
from jobspy import scrape_jobs
|
from ..jobspy import scrape_jobs
|
||||||
|
|
||||||
|
|
||||||
def test_linkedin():
|
def test_linkedin():
|
||||||
@@ -6,4 +6,4 @@ def test_linkedin():
|
|||||||
site_name="linkedin",
|
site_name="linkedin",
|
||||||
search_term="software engineer",
|
search_term="software engineer",
|
||||||
)
|
)
|
||||||
assert result is not None
|
assert result is not None and result.errors.empty is True
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
from jobspy import scrape_jobs
|
from ..jobspy import scrape_jobs
|
||||||
|
|
||||||
|
|
||||||
def test_ziprecruiter():
|
def test_ziprecruiter():
|
||||||
@@ -7,4 +7,4 @@ def test_ziprecruiter():
|
|||||||
search_term="software engineer",
|
search_term="software engineer",
|
||||||
)
|
)
|
||||||
|
|
||||||
assert result is not None
|
assert result is not None and result.errors.empty is True
|
||||||
|
|||||||
Reference in New Issue
Block a user