Added SimpleCrawler

SimpleCrawler/.dockerfile

@@ -0,0 +1,14 @@
+# Use an official Python runtime as a parent image
+FROM python:3.10-slim
+
+# Set the working directory in the container
+WORKDIR /usr/src/app
+
+# Copy the current directory contents into the container at /usr/src/app
+COPY . .
+
+# Install any needed packages specified in requirements.txt
+RUN pip install --no-cache-dir -r requirements.txt
+
+# Run main.py when the container launches
+CMD ["python", "./main.py"]

SimpleCrawler/.license

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) [2024] [Tobias Tischner]
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE

SimpleCrawler/README.md

@@ -0,0 +1,54 @@
+SimpleCrawler README
+
+Overview
+
+SimpleCrawler is a Python script designed to perform basic web scraping operations. It navigates through web pages starting from a given URL, following links up to a specified depth, and logs the content of each visited page. The script is built to handle concurrent requests using threading, which speeds up the scraping process.
+
+Features
+Configurable starting URL and scraping depth
+Thread-safe set modification to track visited URLs
+Concurrent scraping with ThreadPoolExecutor
+Session retries with backoff for handling request failures
+Proxy support through environment variables
+User-Agent customization for HTTP requests
+Logging of scraping activities and errors
+Requirements
+Python 3.x
+requests library
+beautifulsoup4 library
+concurrent.futures module (included in Python 3.2 and above)
+dotenv library (optional for loading environment variables)
+
+Installation
+Before running the script, ensure you have Python 3 installed on your system. You can then install the required Python libraries using pip:
+
+
+
+pip install requests beautifulsoup4 python-dotenv
+
+
+Usage
+Set the start_url variable in the main function to the URL you want to start scraping from.
+(Optional) Create a .env file in the same directory as the script to define HTTP_PROXY and HTTPS_PROXY if you are behind a proxy.
+Run the script using the following command:
+
+
+python app.py
+
+
+Configuration
+You can configure the following settings in the SimpleCrawler class:
+
+start_url: The URL where the crawler will begin scraping.
+max_depth: The maximum depth of links the crawler will follow from the starting URL.
+max_threads: The maximum number of threads to use for concurrent scraping.
+Logging
+The script uses Python's built-in logging module to log its activities. By default, it logs informational messages to the console. You can adjust the logging level and format by modifying the logging.basicConfig call.
+
+Disclaimer
+Web scraping can be against the terms of service of some websites. Always check the website's robots.txt file and terms of service before scraping. Use this script responsibly and ethically.
+
+License
+This script is provided "as is", without warranty of any kind. You may use, modify, and distribute it under the terms of the MIT License.
+
+This README provides a basic introduction to the SimpleCrawler script. For more detailed information on the implementation and customization, refer to the script's source code and comments.

SimpleCrawler/app.py

@@ -0,0 +1,89 @@
+import os
+import logging
+import requests
+from bs4 import BeautifulSoup
+from urllib.parse import urljoin
+from concurrent.futures import ThreadPoolExecutor
+from requests.exceptions import RequestException
+from requests.adapters import HTTPAdapter
+from urllib3.util.retry import Retry
+from dotenv import load_dotenv
+from threading import Lock
+
+# Load environment variables from .env file
+load_dotenv()
+
+# Configure logging
+logging.basicConfig(
+    format='%(asctime)s - %(levelname)s - %(message)s',
+    datefmt='%Y-%m-%d %H:%M:%S',
+    level=logging.INFO
+)
+
+class SimpleCrawler:
+    def __init__(self, start_url, max_depth=3, max_threads=5):
+        self.start_url = start_url
+        self.max_depth = max_depth
+        self.visited = set()
+        self.max_threads = max_threads
+        self.session = self.create_session()
+        self.lock = Lock()  # Lock for thread-safe set modification
+
+    def create_session(self):
+        session = requests.Session()
+        session.headers.update({'User-Agent': 'SimpleCrawler/1.0'})
+
+        # Load proxy settings from environment variables
+        http_proxy = os.getenv('HTTP_PROXY')
+        https_proxy = os.getenv('HTTPS_PROXY')
+        proxies = {
+            'http': http_proxy,
+            'https': https_proxy,
+        }
+        if http_proxy or https_proxy:
+            session.proxies.update(proxies)
+
+        retries = Retry(
+            total=5,
+            backoff_factor=1,
+            status_forcelist=[500, 502, 503, 504]
+        )
+        session.mount('http://', HTTPAdapter(max_retries=retries))
+        session.mount('https://', HTTPAdapter(max_retries=retries))
+        return session
+
+    def scrape_website(self, url, depth=0):
+        if depth > self.max_depth:
+            return
+        with self.lock:  # Acquire lock before modifying the set
+            if url in self.visited:
+                return
+            self.visited.add(url)
+        logging.info(f"Scraping {url}")
+
+        try:
+            response = self.session.get(url)
+            response.raise_for_status()
+
+            soup = BeautifulSoup(response.text, 'html.parser')
+            plain_text = soup.get_text(separator='\n', strip=True)
+            logging.debug(f"Content from {url}:\n{plain_text}\n")  # Changed to debug level
+
+            links = [urljoin(url, link['href']) for link in soup.find_all('a', href=True)]
+            links = [link for link in links if link not in self.visited]  # Removed redundant urljoin
+
+            with ThreadPoolExecutor(max_workers=self.max_threads) as executor:
+                futures = [executor.submit(self.scrape_website, link, depth + 1) for link in links]
+                for future in futures:
+                    future.result()
+
+        except RequestException as e:
+            logging.error(f"An error occurred while scraping {url}: {e}")
+
+def main():
+    start_url = 'https://developers.codegpt.co/'  # Replace with your target URL
+    crawler = SimpleCrawler(start_url, max_depth=3, max_threads=10)
+    crawler.scrape_website(start_url)
+
+if __name__ == "__main__":
+    main()