import csv
import requests
from bs4 import BeautifulSoup
def scrape_page(url):
# Send a GET request to the URL
response = requests.get(url)
# Check if the page exists
if response.status_code != 200:
return []
# Parse the HTML content
soup = BeautifulSoup(response.content, "html.parser")
# Find the table containing the stock names
table = soup.find("table", class_="data-table")
if table:
# Find all rows in the table
rows = table.find_all("tr")
# Extract stock names from each row
stock_names = []
for row in rows[1:]: # Exclude header row
cells = row.find_all("td")
if cells: # Check if cells exist in the row
stock_name = cells[1].text.strip() # Assuming the stock name is in the first column
stock_names.append(stock_name)
return stock_names
else:
print("Table not found on page:", url)
return []
def scrape_all_pages(base_url):
all_stock_names = []
# Iterate over each page
page_number = 1
while True:
page_url = base_url.format(page_number)
print("Scraping page:", page_url)
stock_names = scrape_page(page_url)
if not stock_names:
break
all_stock_names.extend(stock_names)
page_number += 1
return all_stock_names
# URL of the webpage with pagination
base_url = "https://www.screener.in/screens/198573/all-stock-list-bse/?page={}"
# Scrape data from all pages
all_stock_names = scrape_all_pages(base_url)
# Write the stock names to a CSV file
csv_filename = "stock_names.csv"
with open(csv_filename, "w", newline="", encoding="utf-8") as csv_file:
csv_writer = csv.writer(csv_file)
csv_writer.writerow(["Stock Name"]) # Write header
for name in all_stock_names:
csv_writer.writerow([name])
print("All stock names have been scraped and saved in", csv_filename)