v1

visit.py

@@ -0,0 +1,172 @@
+from selenium import webdriver
+from selenium.webdriver.chrome.options import Options
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support.ui import WebDriverWait
+from selenium.webdriver.support import expected_conditions as EC
+from selenium.common.exceptions import TimeoutException, NoSuchElementException, WebDriverException
+import time
+import os
+
+# ========================= CONFIG =========================
+PROXY_IP = "192.168.0.38"
+PROXY_PORT = "1080"
+DOWNLOAD_PATH = os.path.join(os.getcwd(), "downloads")
+LOG_FILE = "download_log.txt"
+
+os.makedirs(DOWNLOAD_PATH, exist_ok=True)
+
+# ====================== CHROME OPTIONS (Optimized for speed) ======================
+chrome_options = Options()
+
+# Proxy & basic stealth
+chrome_options.add_argument(f'--proxy-server=socks5://{PROXY_IP}:{PROXY_PORT}')
+chrome_options.add_argument("user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Safari/537.36")
+chrome_options.add_argument("--no-sandbox")
+chrome_options.add_argument("--disable-dev-shm-usage")
+chrome_options.add_argument("--disable-blink-features=AutomationControlled")
+chrome_options.add_experimental_option("excludeSwitches", ["enable-automation"])
+chrome_options.add_experimental_option('useAutomationExtension', False)
+
+# === SPEED OPTIMIZATIONS ===
+chrome_options.add_argument("--blink-settings=imagesEnabled=false")   # Disable images
+chrome_options.add_argument("--disable-gpu")
+chrome_options.add_argument("--disable-extensions")
+chrome_options.add_argument("--disable-plugins")
+chrome_options.add_argument("--disable-popup-blocking")
+
+# Fast page load strategy (doesn't wait for images/media)
+chrome_options.page_load_strategy = "eager"   # or "none" if you want even faster (but less stable)
+
+# Block media, stylesheets, fonts, etc.
+prefs = {
+    "download.default_directory": DOWNLOAD_PATH,
+    "download.prompt_for_download": False,
+    "download.directory_upgrade": True,
+    "safebrowsing.enabled": True,
+    "profile.default_content_setting_values.automatic_downloads": 1,
+    "profile.managed_default_content_settings.images": 2,        # 2 = block images
+    "profile.managed_default_content_settings.stylesheets": 2,   # block CSS
+    "profile.managed_default_content_settings.fonts": 2,         # block fonts
+    "profile.managed_default_content_settings.media_stream": 2,  # block video/audio
+}
+
+chrome_options.add_experimental_option("prefs", prefs)
+
+# Optional: Run headless (much faster, recommended)
+# chrome_options.add_argument("--headless=new")
+
+# Initialize driver
+driver = webdriver.Chrome(options=chrome_options)
+
+def wait_for_page_load(driver, timeout=20):
+    """Faster page load wait"""
+    try:
+        WebDriverWait(driver, timeout).until(
+            lambda d: d.execute_script("return document.readyState") in ["interactive", "complete"]
+        )
+    except TimeoutException:
+        print("   → Page load timed out, continuing anyway...")
+
+def is_download_finished(download_path, timeout=120, check_interval=2):
+    """Slightly faster download checker"""
+    end_time = time.time() + timeout
+    while time.time() < end_time:
+        partial_files = [f for f in os.listdir(download_path) if f.endswith(('.crdownload', '.part', '.tmp'))]
+        completed_files = [f for f in os.listdir(download_path) if not f.endswith(('.crdownload', '.part', '.tmp'))]
+
+        if completed_files and not partial_files:
+            return True, completed_files
+
+        if partial_files:
+            print(f"   → Still downloading... ({len(partial_files)} partial)")
+        time.sleep(check_interval)
+    return False, []
+
+def clear_partial_downloads():
+    for filename in os.listdir(DOWNLOAD_PATH):
+        if filename.endswith(('.crdownload', '.part', '.tmp')):
+            try:
+                os.remove(os.path.join(DOWNLOAD_PATH, filename))
+            except:
+                pass
+
+def log_result(url: str, success: bool):
+    status = "True" if success else "False"
+    with open(LOG_FILE, "a", encoding="utf-8") as f:
+        f.write(f"{url} | {status}\n")
+    print(f"   Logged: {'SUCCESS' if success else 'FAILED'}")
+
+def run_download(url: str):
+    try:
+        print(f"\n[{time.strftime('%H:%M:%S')}] Processing → {url}")
+        clear_partial_downloads()
+
+        print("   Navigating...")
+        driver.get(url)
+
+        wait_for_page_load(driver, timeout=20)
+        print("   ✓ Page loaded (eager).")
+
+        wait = WebDriverWait(driver, 10)  # Reduced timeout
+
+        # === License checkbox (quick check) ===
+        try:
+            checkbox = wait.until(EC.presence_of_element_located((By.NAME, "license_agree")))
+            if checkbox.is_enabled() and checkbox.is_displayed():
+                checkbox.click()
+                print("   ✓ License checkbox accepted.")
+                time.sleep(0.5)
+        except (TimeoutException, NoSuchElementException):
+            print("   → No checkbox found. Continuing...")
+
+        # === Download button (quick detection) ===
+        try:
+            download_btn = wait.until(EC.element_to_be_clickable((By.ID, "jd_license_submit")))
+            download_btn.click()
+            print("   ✓ Download button clicked.")
+        except (TimeoutException, NoSuchElementException):
+            print("   ✗ Download button NOT found or not clickable → Skipping")
+            log_result(url, False)
+            return False
+
+        # === Wait for download ===
+        print("   Waiting for download (max 2 min)...")
+        success, files = is_download_finished(DOWNLOAD_PATH, timeout=120)
+        if success and files:
+            print(f"   ✓ Download completed! Files: {files}")
+            log_result(url, True)
+            return True
+        else:
+            print("   ✗ Download timed out.")
+            log_result(url, False)
+            return False
+
+    except WebDriverException as e:
+        print(f"   ✗ WebDriver error: {e}")
+        log_result(url, False)
+        return False
+    except Exception as e:
+        print(f"   ✗ Unexpected error: {e}")
+        log_result(url, False)
+        return False
+
+# ====================== MAIN ======================
+if __name__ == "__main__":
+    with open("url-final.txt", "r", encoding="utf-8") as f:
+        links = [line.strip() for line in f if line.strip()]
+
+    links = links[250:]  # start from 251st
+    print(f"Loaded {len(links)} URLs. Starting from index 250.")
+
+    success_count = 0
+    for i, url in enumerate(links, 1):
+        print(f"--- [{i}/{len(links)}] ---")
+        if run_download(url):
+            success_count += 1
+        time.sleep(3)  # Reduced delay between requests
+
+    print(f"\n=== FINISHED ===")
+    print(f"Total: {len(links)} | Success: {success_count} | Failed: {len(links) - success_count}")
+    print(f"Log: {LOG_FILE}")
+
+    driver.quit()