sttil/skool-lesson-scrape
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Fix unterminated string literals breaking scrape.py entirely

Browse source 
Six print/write_text calls had literal control characters (CR, tab, or
a raw newline) embedded inside single-line string literals instead of
escaped \r\n\t sequences. This was a hard SyntaxError in any Python 3
version, so the script could never have actually run as committed.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01ViRGe7WHuUXUpHqvUfuW5r
This commit is contained in:
sososttil 2026-07-26 09:12:15 -04:00
parent aa33c361c2
commit 7d16130712
1 changed files with 7 additions and 15 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

22
scrape.py
View file

@ -36,8 +36,7 @@ CONTENT_SELECTORS = [
def sanitize(name: str) -> str: def sanitize(name: str) -> str:
name = re.sub(r'[<>:"/\|?* name = re.sub(r'[<>:"/\\|?*\r\n\t]', '', str(name)).strip().strip(".")
]', '', str(name)).strip().strip(".")
return name[:120] return name[:120]
@ -62,9 +61,7 @@ def html_to_md(raw: str) -> str:
def write_lesson(output_dir: Path, course_title: str, lesson_title: str, body: str) -> str: def write_lesson(output_dir: Path, course_title: str, lesson_title: str, body: str) -> str:
stem = f"{sanitize(course_title)} -- {sanitize(lesson_title)}" stem = f"{sanitize(course_title)} -- {sanitize(lesson_title)}"
out = output_dir / f"{stem}.md" out = output_dir / f"{stem}.md"
out.write_text(f"# {lesson_title} out.write_text(f"# {lesson_title}\n\n{body}", encoding="utf-8")
{body}", encoding="utf-8")
return stem return stem
@ -84,8 +81,7 @@ async def run(community: str, output_dir: Path, discover: bool = False):
existing = existing_stems(output_dir) existing = existing_stems(output_dir)
print(f"Community: {community}") print(f"Community: {community}")
print(f"Output: {output_dir}") print(f"Output: {output_dir}")
print(f"Lessons already saved: {len(existing)} print(f"Lessons already saved: {len(existing)}\n")
")
async with async_playwright() as p: async with async_playwright() as p:
browser = await p.chromium.launch(headless=False, slow_mo=25) browser = await p.chromium.launch(headless=False, slow_mo=25)
@ -93,12 +89,10 @@ async def run(community: str, output_dir: Path, discover: bool = False):
page = await ctx.new_page() page = await ctx.new_page()
print("Opening Skool — please log in when the browser window appears.") print("Opening Skool — please log in when the browser window appears.")
print("The script will continue automatically once you land on the community. print("The script will continue automatically once you land on the community.\n")
")
await page.goto("https://www.skool.com/login") await page.goto("https://www.skool.com/login")
await page.wait_for_url(f"**/{community}/**", timeout=300_000) await page.wait_for_url(f"**/{community}/**", timeout=300_000)
print("Logged in. print("Logged in.\n")
")
await page.goto(classroom) await page.goto(classroom)
await page.wait_for_load_state("load") await page.wait_for_load_state("load")
@ -107,8 +101,7 @@ async def run(community: str, output_dir: Path, discover: bool = False):
nd = next_data(await page.content()) nd = next_data(await page.content())
all_crses = nd.get("props", {}).get("pageProps", {}).get("allCourses", []) all_crses = nd.get("props", {}).get("pageProps", {}).get("allCourses", [])
courses = [c for c in all_crses if c.get("metadata", {}).get("hasAccess", 0)] courses = [c for c in all_crses if c.get("metadata", {}).get("hasAccess", 0)]
print(f"Accessible courses: {len(courses)} of {len(all_crses)} total print(f"Accessible courses: {len(courses)} of {len(all_crses)} total\n")
")
if not courses: if not courses:
DIAG_DIR.mkdir(parents=True, exist_ok=True) DIAG_DIR.mkdir(parents=True, exist_ok=True)
@ -158,8 +151,7 @@ async def run(community: str, output_dir: Path, discover: bool = False):
) )
if not children: if not children:
print(" No lessons found — skipping print(" No lessons found — skipping\n")
")
continue continue
print(f" {len(children)} lessons") print(f" {len(children)} lessons")