From 29c407d1e0d8fe06fb4d25f9f23f6aacc7fcd34f Mon Sep 17 00:00:00 2001
From: Tobi Dotcom <121991626+tobidotcom@users.noreply.github.com>
Date: Mon, 1 Apr 2024 22:11:37 +0200
Subject: [PATCH 1/2] Added Dev Container Folder

---
 .devcontainer/devcontainer.json | 33 +++++++++++++++++++++++++++++++++
 1 file changed, 33 insertions(+)
 create mode 100644 .devcontainer/devcontainer.json

diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json
new file mode 100644
index 0000000..ff35faf
--- /dev/null
+++ b/.devcontainer/devcontainer.json
@@ -0,0 +1,33 @@
+{
+  "name": "Python 3",
+  // Or use a Dockerfile or Docker Compose file. More info: https://containers.dev/guide/dockerfile
+  "image": "mcr.microsoft.com/devcontainers/python:1-3.11-bullseye",
+  "customizations": {
+    "codespaces": {
+      "openFiles": [
+        "README.md",
+        "examples/streamlit_medium_writer_chat/app.py"
+      ]
+    },
+    "vscode": {
+      "settings": {},
+      "extensions": [
+        "ms-python.python",
+        "ms-python.vscode-pylance"
+      ]
+    }
+  },
+  "updateContentCommand": "[ -f packages.txt ] && sudo apt update && sudo apt upgrade -y && sudo xargs apt install -y <packages.txt; [ -f requirements.txt ] && pip3 install --user -r requirements.txt; pip3 install --user streamlit; echo '✅ Packages installed and Requirements met'",
+  "postAttachCommand": {
+    "server": "streamlit run examples/streamlit_medium_writer_chat/app.py --server.enableCORS false --server.enableXsrfProtection false"
+  },
+  "portsAttributes": {
+    "8501": {
+      "label": "Application",
+      "onAutoForward": "openPreview"
+    }
+  },
+  "forwardPorts": [
+    8501
+  ]
+}
\ No newline at end of file

From 87e93ba18769a9168d332d1c446463bc65403e59 Mon Sep 17 00:00:00 2001
From: Tobi Dotcom <121991626+tobidotcom@users.noreply.github.com>
Date: Tue, 2 Apr 2024 22:43:50 +0000
Subject: [PATCH 2/2] Developed SimpleCrawler - an Ethical Crawler with several
 features

---
 .devcontainer/devcontainer.json |  33 ------------
 SimpleCrawler/.dockerfile       |  14 +++++
 SimpleCrawler/.license          |  21 ++++++++
 SimpleCrawler/README.md         |  54 +++++++++++++++++++
 SimpleCrawler/app.py            |  89 ++++++++++++++++++++++++++++++++
 SimpleCrawler/requirements.txt  | Bin 0 -> 308 bytes
 6 files changed, 178 insertions(+), 33 deletions(-)
 delete mode 100644 .devcontainer/devcontainer.json
 create mode 100644 SimpleCrawler/.dockerfile
 create mode 100644 SimpleCrawler/.license
 create mode 100644 SimpleCrawler/README.md
 create mode 100644 SimpleCrawler/app.py
 create mode 100644 SimpleCrawler/requirements.txt

diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json
deleted file mode 100644
index ff35faf..0000000
--- a/.devcontainer/devcontainer.json
+++ /dev/null
@@ -1,33 +0,0 @@
-{
-  "name": "Python 3",
-  // Or use a Dockerfile or Docker Compose file. More info: https://containers.dev/guide/dockerfile
-  "image": "mcr.microsoft.com/devcontainers/python:1-3.11-bullseye",
-  "customizations": {
-    "codespaces": {
-      "openFiles": [
-        "README.md",
-        "examples/streamlit_medium_writer_chat/app.py"
-      ]
-    },
-    "vscode": {
-      "settings": {},
-      "extensions": [
-        "ms-python.python",
-        "ms-python.vscode-pylance"
-      ]
-    }
-  },
-  "updateContentCommand": "[ -f packages.txt ] && sudo apt update && sudo apt upgrade -y && sudo xargs apt install -y <packages.txt; [ -f requirements.txt ] && pip3 install --user -r requirements.txt; pip3 install --user streamlit; echo '✅ Packages installed and Requirements met'",
-  "postAttachCommand": {
-    "server": "streamlit run examples/streamlit_medium_writer_chat/app.py --server.enableCORS false --server.enableXsrfProtection false"
-  },
-  "portsAttributes": {
-    "8501": {
-      "label": "Application",
-      "onAutoForward": "openPreview"
-    }
-  },
-  "forwardPorts": [
-    8501
-  ]
-}
\ No newline at end of file
diff --git a/SimpleCrawler/.dockerfile b/SimpleCrawler/.dockerfile
new file mode 100644
index 0000000..13aa5a5
--- /dev/null
+++ b/SimpleCrawler/.dockerfile
@@ -0,0 +1,14 @@
+# Use an official Python runtime as a parent image
+FROM python:3.10-slim
+
+# Set the working directory in the container
+WORKDIR /usr/src/app
+
+# Copy the current directory contents into the container at /usr/src/app
+COPY . .
+
+# Install any needed packages specified in requirements.txt
+RUN pip install --no-cache-dir -r requirements.txt
+
+# Run main.py when the container launches
+CMD ["python", "./main.py"]
\ No newline at end of file
diff --git a/SimpleCrawler/.license b/SimpleCrawler/.license
new file mode 100644
index 0000000..3a87760
--- /dev/null
+++ b/SimpleCrawler/.license
@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) [2024] [Tobias Tischner]
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE
\ No newline at end of file
diff --git a/SimpleCrawler/README.md b/SimpleCrawler/README.md
new file mode 100644
index 0000000..f09d818
--- /dev/null
+++ b/SimpleCrawler/README.md
@@ -0,0 +1,54 @@
+SimpleCrawler README
+
+Overview
+
+SimpleCrawler is a Python script designed to perform basic web scraping operations. It navigates through web pages starting from a given URL, following links up to a specified depth, and logs the content of each visited page. The script is built to handle concurrent requests using threading, which speeds up the scraping process.
+
+Features
+Configurable starting URL and scraping depth
+Thread-safe set modification to track visited URLs
+Concurrent scraping with ThreadPoolExecutor
+Session retries with backoff for handling request failures
+Proxy support through environment variables
+User-Agent customization for HTTP requests
+Logging of scraping activities and errors
+Requirements
+Python 3.x
+requests library
+beautifulsoup4 library
+concurrent.futures module (included in Python 3.2 and above)
+dotenv library (optional for loading environment variables)
+
+Installation
+Before running the script, ensure you have Python 3 installed on your system. You can then install the required Python libraries using pip:
+
+
+
+pip install requests beautifulsoup4 python-dotenv
+
+
+Usage
+Set the start_url variable in the main function to the URL you want to start scraping from.
+(Optional) Create a .env file in the same directory as the script to define HTTP_PROXY and HTTPS_PROXY if you are behind a proxy.
+Run the script using the following command:
+
+
+python app.py
+
+
+Configuration
+You can configure the following settings in the SimpleCrawler class:
+
+start_url: The URL where the crawler will begin scraping.
+max_depth: The maximum depth of links the crawler will follow from the starting URL.
+max_threads: The maximum number of threads to use for concurrent scraping.
+Logging
+The script uses Python's built-in logging module to log its activities. By default, it logs informational messages to the console. You can adjust the logging level and format by modifying the logging.basicConfig call.
+
+Disclaimer
+Web scraping can be against the terms of service of some websites. Always check the website's robots.txt file and terms of service before scraping. Use this script responsibly and ethically.
+
+License
+This script is provided "as is", without warranty of any kind. You may use, modify, and distribute it under the terms of the MIT License.
+
+This README provides a basic introduction to the SimpleCrawler script. For more detailed information on the implementation and customization, refer to the script's source code and comments.
\ No newline at end of file
diff --git a/SimpleCrawler/app.py b/SimpleCrawler/app.py
new file mode 100644
index 0000000..13b8d53
--- /dev/null
+++ b/SimpleCrawler/app.py
@@ -0,0 +1,89 @@
+import os
+import logging
+import requests
+from bs4 import BeautifulSoup
+from urllib.parse import urljoin
+from concurrent.futures import ThreadPoolExecutor
+from requests.exceptions import RequestException
+from requests.adapters import HTTPAdapter
+from urllib3.util.retry import Retry
+from dotenv import load_dotenv
+from threading import Lock
+
+# Load environment variables from .env file
+load_dotenv()
+
+# Configure logging
+logging.basicConfig(
+    format='%(asctime)s - %(levelname)s - %(message)s',
+    datefmt='%Y-%m-%d %H:%M:%S',
+    level=logging.INFO
+)
+
+class SimpleCrawler:
+    def __init__(self, start_url, max_depth=3, max_threads=5):
+        self.start_url = start_url
+        self.max_depth = max_depth
+        self.visited = set()
+        self.max_threads = max_threads
+        self.session = self.create_session()
+        self.lock = Lock()  # Lock for thread-safe set modification
+
+    def create_session(self):
+        session = requests.Session()
+        session.headers.update({'User-Agent': 'SimpleCrawler/1.0'})
+
+        # Load proxy settings from environment variables
+        http_proxy = os.getenv('HTTP_PROXY')
+        https_proxy = os.getenv('HTTPS_PROXY')
+        proxies = {
+            'http': http_proxy,
+            'https': https_proxy,
+        }
+        if http_proxy or https_proxy:
+            session.proxies.update(proxies)
+
+        retries = Retry(
+            total=5,
+            backoff_factor=1,
+            status_forcelist=[500, 502, 503, 504]
+        )
+        session.mount('http://', HTTPAdapter(max_retries=retries))
+        session.mount('https://', HTTPAdapter(max_retries=retries))
+        return session
+
+    def scrape_website(self, url, depth=0):
+        if depth > self.max_depth:
+            return
+        with self.lock:  # Acquire lock before modifying the set
+            if url in self.visited:
+                return
+            self.visited.add(url)
+        logging.info(f"Scraping {url}")
+
+        try:
+            response = self.session.get(url)
+            response.raise_for_status()
+
+            soup = BeautifulSoup(response.text, 'html.parser')
+            plain_text = soup.get_text(separator='\n', strip=True)
+            logging.debug(f"Content from {url}:\n{plain_text}\n")  # Changed to debug level
+
+            links = [urljoin(url, link['href']) for link in soup.find_all('a', href=True)]
+            links = [link for link in links if link not in self.visited]  # Removed redundant urljoin
+
+            with ThreadPoolExecutor(max_workers=self.max_threads) as executor:
+                futures = [executor.submit(self.scrape_website, link, depth + 1) for link in links]
+                for future in futures:
+                    future.result()
+
+        except RequestException as e:
+            logging.error(f"An error occurred while scraping {url}: {e}")
+
+def main():
+    start_url = 'https://developers.codegpt.co/'  # Replace with your target URL
+    crawler = SimpleCrawler(start_url, max_depth=3, max_threads=10)
+    crawler.scrape_website(start_url)
+
+if __name__ == "__main__":
+    main()
\ No newline at end of file
diff --git a/SimpleCrawler/requirements.txt b/SimpleCrawler/requirements.txt
new file mode 100644
index 0000000000000000000000000000000000000000..807edaead20892d5548d875a8d9aa643a4f6a5c5
GIT binary patch
literal 308
zcmX|++YW*-5CzY(iJzk3rf+@>N)054SinR-UY%XQH0>tc>FnwEb5>BPNuwHFsjUXx
zsl5uFScUdVm2u|UYNHE$T<FG$ObLDCjLH_O#u{-2*qu6@Q8(^d=Dgiu_T0Vet40-M
z=~n9;-PP-bqsGu-s;DFHJ3XCC;SzT0Y^3ubePHs)wi$EwnX@0P{~JgWu!KK#aK}W_
O)^nc8w;Iec&-ekqz%OV3

literal 0
HcmV?d00001