Update UI, and Update Sitemap tool to get proper thread count
Build Docker Image / docker (push) Successful in 6s

This commit is contained in:
2026-04-09 11:27:13 -07:00
parent 0e410a1f6c
commit 287566716f
5 changed files with 32 additions and 5 deletions
+9 -3
View File
@@ -61,6 +61,8 @@ def render_sitemap_tab() -> None:
st.caption("Crawl a site, export a sitemap CSV, and keep resume data inside the container data volume.")
SITEMAP_OUTPUT_DIR.mkdir(parents=True, exist_ok=True)
sitemap_builder = get_sitemap_module()
default_workers = sitemap_builder.DEFAULT_WORKERS
with st.form("sitemap-form"):
start_url = st.text_input("Starting URL", placeholder="https://example.com")
@@ -73,7 +75,13 @@ def render_sitemap_tab() -> None:
col1, col2, col3 = st.columns(3)
with col1:
max_pages = st.number_input("Max pages", min_value=1, value=10000, step=100)
workers = st.number_input("Worker threads", min_value=1, value=8, step=1)
workers = st.number_input(
"Worker threads",
min_value=1,
value=default_workers,
step=1,
help="Defaults to the number of CPUs visible inside the Docker container.",
)
with col2:
delay = st.number_input("Delay between requests (seconds)", min_value=0.0, value=0.0, step=0.25)
timeout = st.number_input("Request timeout (seconds)", min_value=1.0, value=15.0, step=1.0)
@@ -90,7 +98,6 @@ def render_sitemap_tab() -> None:
if not start_url.strip():
st.error("Starting URL is required.")
else:
sitemap_builder = get_sitemap_module()
safe_name = sanitize_job_name(job_name)
output_path = SITEMAP_OUTPUT_DIR / f"{safe_name}.csv"
captured_stdout = io.StringIO()
@@ -137,7 +144,6 @@ def render_sitemap_tab() -> None:
st.info("Run a crawl to generate a sitemap CSV.")
return
sitemap_builder = get_sitemap_module()
summary = result_data["summary"]
csv_path = Path(result_data["output_path"])
state_path = Path(result_data["state_path"])