Compare commits

..

3 commits

Author SHA1 Message Date
sososttil
f12ab4e3a0 Add .gitignore for venv
Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01ViRGe7WHuUXUpHqvUfuW5r
2026-07-26 10:53:49 -04:00
sososttil
f0479d604a Fix login-detection glob requiring a path segment after the community
wait_for_url("**/{community}/**") requires characters both before and
after the community slug. The community home page itself is exactly
https://www.skool.com/<community> with nothing trailing, so the first
real run navigated there correctly (confirmed in the logs: chase-ai
redirect, then /navaigate) but still timed out waiting for a pattern
that could never match. Switched to a plain substring predicate.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01ViRGe7WHuUXUpHqvUfuW5r
2026-07-26 09:18:40 -04:00
sososttil
7d16130712 Fix unterminated string literals breaking scrape.py entirely
Six print/write_text calls had literal control characters (CR, tab, or
a raw newline) embedded inside single-line string literals instead of
escaped \r\n\t sequences. This was a hard SyntaxError in any Python 3
version, so the script could never have actually run as committed.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01ViRGe7WHuUXUpHqvUfuW5r
2026-07-26 09:12:15 -04:00
2 changed files with 11 additions and 16 deletions

3
.gitignore vendored Normal file
View file

@ -0,0 +1,3 @@
venv/
__pycache__/
*.pyc

View file

@ -36,8 +36,7 @@ CONTENT_SELECTORS = [
def sanitize(name: str) -> str:
name = re.sub(r'[<>:"/\|?*
]', '', str(name)).strip().strip(".")
name = re.sub(r'[<>:"/\\|?*\r\n\t]', '', str(name)).strip().strip(".")
return name[:120]
@ -62,9 +61,7 @@ def html_to_md(raw: str) -> str:
def write_lesson(output_dir: Path, course_title: str, lesson_title: str, body: str) -> str:
stem = f"{sanitize(course_title)} -- {sanitize(lesson_title)}"
out = output_dir / f"{stem}.md"
out.write_text(f"# {lesson_title}
{body}", encoding="utf-8")
out.write_text(f"# {lesson_title}\n\n{body}", encoding="utf-8")
return stem
@ -84,8 +81,7 @@ async def run(community: str, output_dir: Path, discover: bool = False):
existing = existing_stems(output_dir)
print(f"Community: {community}")
print(f"Output: {output_dir}")
print(f"Lessons already saved: {len(existing)}
")
print(f"Lessons already saved: {len(existing)}\n")
async with async_playwright() as p:
browser = await p.chromium.launch(headless=False, slow_mo=25)
@ -93,12 +89,10 @@ async def run(community: str, output_dir: Path, discover: bool = False):
page = await ctx.new_page()
print("Opening Skool — please log in when the browser window appears.")
print("The script will continue automatically once you land on the community.
")
print("The script will continue automatically once you land on the community.\n")
await page.goto("https://www.skool.com/login")
await page.wait_for_url(f"**/{community}/**", timeout=300_000)
print("Logged in.
")
await page.wait_for_url(lambda url: f"/{community}" in url, timeout=300_000)
print("Logged in.\n")
await page.goto(classroom)
await page.wait_for_load_state("load")
@ -107,8 +101,7 @@ async def run(community: str, output_dir: Path, discover: bool = False):
nd = next_data(await page.content())
all_crses = nd.get("props", {}).get("pageProps", {}).get("allCourses", [])
courses = [c for c in all_crses if c.get("metadata", {}).get("hasAccess", 0)]
print(f"Accessible courses: {len(courses)} of {len(all_crses)} total
")
print(f"Accessible courses: {len(courses)} of {len(all_crses)} total\n")
if not courses:
DIAG_DIR.mkdir(parents=True, exist_ok=True)
@ -158,8 +151,7 @@ async def run(community: str, output_dir: Path, discover: bool = False):
)
if not children:
print(" No lessons found — skipping
")
print(" No lessons found — skipping\n")
continue
print(f" {len(children)} lessons")