Update UI, and Update Sitemap tool to get proper thread count
Build Docker Image / docker (push) Successful in 6s
Build Docker Image / docker (push) Successful in 6s
This commit is contained in:
@@ -61,6 +61,8 @@ def render_sitemap_tab() -> None:
|
||||
st.caption("Crawl a site, export a sitemap CSV, and keep resume data inside the container data volume.")
|
||||
|
||||
SITEMAP_OUTPUT_DIR.mkdir(parents=True, exist_ok=True)
|
||||
sitemap_builder = get_sitemap_module()
|
||||
default_workers = sitemap_builder.DEFAULT_WORKERS
|
||||
|
||||
with st.form("sitemap-form"):
|
||||
start_url = st.text_input("Starting URL", placeholder="https://example.com")
|
||||
@@ -73,7 +75,13 @@ def render_sitemap_tab() -> None:
|
||||
col1, col2, col3 = st.columns(3)
|
||||
with col1:
|
||||
max_pages = st.number_input("Max pages", min_value=1, value=10000, step=100)
|
||||
workers = st.number_input("Worker threads", min_value=1, value=8, step=1)
|
||||
workers = st.number_input(
|
||||
"Worker threads",
|
||||
min_value=1,
|
||||
value=default_workers,
|
||||
step=1,
|
||||
help="Defaults to the number of CPUs visible inside the Docker container.",
|
||||
)
|
||||
with col2:
|
||||
delay = st.number_input("Delay between requests (seconds)", min_value=0.0, value=0.0, step=0.25)
|
||||
timeout = st.number_input("Request timeout (seconds)", min_value=1.0, value=15.0, step=1.0)
|
||||
@@ -90,7 +98,6 @@ def render_sitemap_tab() -> None:
|
||||
if not start_url.strip():
|
||||
st.error("Starting URL is required.")
|
||||
else:
|
||||
sitemap_builder = get_sitemap_module()
|
||||
safe_name = sanitize_job_name(job_name)
|
||||
output_path = SITEMAP_OUTPUT_DIR / f"{safe_name}.csv"
|
||||
captured_stdout = io.StringIO()
|
||||
@@ -137,7 +144,6 @@ def render_sitemap_tab() -> None:
|
||||
st.info("Run a crawl to generate a sitemap CSV.")
|
||||
return
|
||||
|
||||
sitemap_builder = get_sitemap_module()
|
||||
summary = result_data["summary"]
|
||||
csv_path = Path(result_data["output_path"])
|
||||
state_path = Path(result_data["state_path"])
|
||||
|
||||
Reference in New Issue
Block a user