|
|
@@ -53,16 +53,19 @@ class ScraperThread(QThread):
|
|
|
scraper = Scraper1688(headless=self.headless, status_callback=status_cb, log_callback=self.log.emit)
|
|
|
collected_count = 0
|
|
|
product_index = initial_p_count
|
|
|
-
|
|
|
+ prev_batch_end = start_time
|
|
|
+
|
|
|
for batch_results in scraper.search_products_yield(self.keyword, total_count=self.total_count, existing_links=existing_links):
|
|
|
+ batch_elapsed = int(time.time() - prev_batch_end)
|
|
|
append_to_template(batch_results, self.output_path, status_callback=status_cb)
|
|
|
unique_links = len(set(item.get('link') for item in batch_results if item.get('link')))
|
|
|
product_index += unique_links
|
|
|
collected_count += len(batch_results)
|
|
|
- self.log.emit(f"[+] 解析到第 {product_index} 个商品，新增数据已持久化: {len(batch_results)} 条，本次共计: {collected_count}")
|
|
|
+ self.log.emit(f"[+] 解析到第 {product_index} 个商品，新增数据已持久化: {len(batch_results)} 条，耗时{batch_elapsed}秒。本次共计: {collected_count}条。")
|
|
|
current_task_done = product_index - initial_p_count
|
|
|
prog = int((current_task_done / self.total_count) * 100)
|
|
|
self.progress.emit(min(prog, 100))
|
|
|
+ prev_batch_end = time.time()
|
|
|
|
|
|
duration = time.time() - start_time
|
|
|
self.log.emit(f"<b>[完成] 任务结束，本次共解析 {product_index - initial_p_count} 个商品。</b>")
|
LuTong