always quit driver and only get lidl sublinks with most value

Author: StefanBratanov

src/main/kotlin/com/stefanbratanov/sofiasupermarketsapi/brochure/FantasticoBrochureDownloader.kt

@@ -54,10 +54,9 @@ class FantasticoBrochureDownloader(@Value("\${fantastico.url}") private val url:
     val htmlDoc = getHtmlDocument(url)
 
     val driver = ChromeDriver(options)
-
     val waitDriver = WebDriverWait(driver, Duration.ofSeconds(30))
 
-    val brochures =
+    return try {
       htmlDoc
         .select("div.brochure-container.first div.hold-options")
         .filter {
@@ -81,6 +80,7 @@ class FantasticoBrochureDownloader(@Value("\${fantastico.url}") private val url:
 
           // loading the flipping book
           for (attempt in 1..3) try {
+            log.info("Loading flipping book from {}", flippingBookUrl)
             driver.get(flippingBookUrl)
             break
           } catch (_: TimeoutException) {
@@ -110,10 +110,9 @@ class FantasticoBrochureDownloader(@Value("\${fantastico.url}") private val url:
 
           Brochure(downloadPath, dateRange?.first, dateRange?.second)
         }
-
-    driver.quit()
-
-    return brochures
+    } finally {
+      driver.quit()
+    }
   }
 
   private fun extractDateRange(description: String?): Pair<LocalDate?, LocalDate?>? {

src/main/kotlin/com/stefanbratanov/sofiasupermarketsapi/links/LidlSublinksScraper.kt

@@ -12,14 +12,21 @@ import org.springframework.stereotype.Component
 @Component
 class LidlSublinksScraper(@Value("\${lidl.base.url}") private val baseUrl: URL) : SublinksScraper {
 
+  private val sublinksToAccept =
+    listOf("niska-tsena-visoko-kachestvo".toRegex(), "lidl-plus".toRegex())
+
   override fun getSublinks(): List<URL> {
     log.info("Scraping {} for sublinks", baseUrl)
 
     return getHtmlDocument(baseUrl)
       .select("li.AHeroStageItems__Item > a")
-      .map {
+      .mapNotNull {
         val href = it.attr("href")
-        baseUrl.toURI().resolve(href).toURL()
+        if (sublinksToAccept.none { rgx -> href.contains(rgx) }) {
+          null
+        } else {
+          baseUrl.toURI().resolve(href).toURL()
+        }
       }
       .distinct()
   }