0.0.99

j-mendez · j-mendez · commit bb49bdb7fecf · 2024-07-19T18:04:44.000-04:00
diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml
diff --git a/.yarnrc.yml b/.yarnrc.yml
@@ -1,5 +1,15 @@
 nodeLinker: node-modules
-
 npmAuditRegistry: "https://registry.npmjs.org"
-
-yarnPath: .yarn/releases/yarn-4.3.1.cjs
+yarnPath: .yarn/releases/yarn-4.3.1.cjs
+supportedArchitectures:
+  cpu:
+    - ia32
+    - x64
+    - arm
+    - arm64
+    - freebsd
+    - macos
+    - windows
+    - linux
+    - gnu
+    - musl
diff --git a/Cargo.toml b/Cargo.toml
@@ -17,16 +17,37 @@ napi-derive = "2"
 num_cpus = "1"
 serde = "1"
 serde_json = "1"
-spider = { version = "1.98.4", features = ["napi", "cron", "regex", "cookies", "socks", "chrome", "control", "chrome_intercept", "cache", "openai", "serde", "real_browser" ] }
+spider = { version = "1.99.5", features = ["napi", "cron", "regex", "cookies", "socks", "chrome", "control", "chrome_intercept", "cache", "openai", "serde", "real_browser", "headers" ] }
 
 [target.x86_64-unknown-linux-gnu.dependencies]
 openssl-sys = { version = "0.9", features = ["vendored"] }
 openssl = { version = "0.10", features = ["vendored"] }
 
+[target.aarch64-unknown-linux-musl.dependencies]
+openssl-sys = { version = "0.9", features = ["vendored"] }
+openssl = { version = "0.10", features = ["vendored"] }
+
+[target.aarch64-unknown-linux-gnu.dependencies]
+openssl-sys = { version = "0.9", features = ["vendored"] }
+openssl = { version = "0.10", features = ["vendored"] }
+
+[target.armv7-unknown-linux-gnueabihf.dependencies]
+openssl-sys = { version = "0.9", features = ["vendored"] }
+openssl = { version = "0.10", features = ["vendored"] }
+
 [target.x86_64-unknown-linux-musl.dependencies]
 openssl-sys = { version = "0.9", features = ["vendored"] }
 openssl = { version = "0.10", features = ["vendored"] }
 
+[target.armv7-linux-androideabi.dependencies]
+openssl-sys = { version = "0.9", features = ["vendored"] }
+openssl = { version = "0.10", features = ["vendored"] }
+
+[target.aarch64-linux-android.dependencies]
+openssl-sys = { version = "0.9", features = ["vendored"] }
+openssl = { version = "0.10", features = ["vendored"] }
+
+
 [build-dependencies]
 napi-build = "2"
 
diff --git a/npm/android-arm-eabi/package.json b/npm/android-arm-eabi/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@spider-rs/spider-rs-android-arm-eabi",
-  "version": "0.0.87",
+  "version": "0.0.139",
   "repository": "https://github.com/spider-rs/spider-nodejs",
   "os": [
     "android"
diff --git a/npm/android-arm64/package.json b/npm/android-arm64/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@spider-rs/spider-rs-android-arm64",
-  "version": "0.0.87",
+  "version": "0.0.139",
   "repository": "https://github.com/spider-rs/spider-nodejs",
   "os": [
     "android"
diff --git a/npm/darwin-arm64/package.json b/npm/darwin-arm64/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@spider-rs/spider-rs-darwin-arm64",
-  "version": "0.0.87",
+  "version": "0.0.139",
   "repository": "https://github.com/spider-rs/spider-nodejs",
   "os": [
     "darwin"
diff --git a/npm/darwin-universal/package.json b/npm/darwin-universal/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@spider-rs/spider-rs-darwin-universal",
-  "version": "0.0.87",
+  "version": "0.0.139",
   "repository": "https://github.com/spider-rs/spider-nodejs",
   "os": [
     "darwin"
diff --git a/npm/darwin-x64/package.json b/npm/darwin-x64/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@spider-rs/spider-rs-darwin-x64",
-  "version": "0.0.87",
+  "version": "0.0.139",
   "repository": "https://github.com/spider-rs/spider-nodejs",
   "os": [
     "darwin"
diff --git a/npm/freebsd-x64/package.json b/npm/freebsd-x64/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@spider-rs/spider-rs-freebsd-x64",
-  "version": "0.0.87",
+  "version": "0.0.139",
   "repository": "https://github.com/spider-rs/spider-nodejs",
   "os": [
     "freebsd"
diff --git a/npm/linux-arm-gnueabihf/package.json b/npm/linux-arm-gnueabihf/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@spider-rs/spider-rs-linux-arm-gnueabihf",
-  "version": "0.0.87",
+  "version": "0.0.139",
   "repository": "https://github.com/spider-rs/spider-nodejs",
   "os": [
     "linux"
diff --git a/npm/linux-arm64-gnu/package.json b/npm/linux-arm64-gnu/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@spider-rs/spider-rs-linux-arm64-gnu",
-  "version": "0.0.87",
+  "version": "0.0.139",
   "repository": "https://github.com/spider-rs/spider-nodejs",
   "os": [
     "linux"
diff --git a/npm/linux-arm64-musl/package.json b/npm/linux-arm64-musl/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@spider-rs/spider-rs-linux-arm64-musl",
-  "version": "0.0.87",
+  "version": "0.0.139",
   "repository": "https://github.com/spider-rs/spider-nodejs",
   "os": [
     "linux"
diff --git a/npm/linux-x64-gnu/package.json b/npm/linux-x64-gnu/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@spider-rs/spider-rs-linux-x64-gnu",
-  "version": "0.0.87",
+  "version": "0.0.139",
   "repository": "https://github.com/spider-rs/spider-nodejs",
   "os": [
     "linux"
diff --git a/npm/linux-x64-musl/package.json b/npm/linux-x64-musl/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@spider-rs/spider-rs-linux-x64-musl",
-  "version": "0.0.87",
+  "version": "0.0.139",
   "repository": "https://github.com/spider-rs/spider-nodejs",
   "os": [
     "linux"
diff --git a/npm/win32-arm64-msvc/package.json b/npm/win32-arm64-msvc/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@spider-rs/spider-rs-win32-arm64-msvc",
-  "version": "0.0.87",
+  "version": "0.0.139",
   "repository": "https://github.com/spider-rs/spider-nodejs",
   "os": [
     "win32"
diff --git a/npm/win32-ia32-msvc/package.json b/npm/win32-ia32-msvc/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@spider-rs/spider-rs-win32-ia32-msvc",
-  "version": "0.0.87",
+  "version": "0.0.139",
   "repository": "https://github.com/spider-rs/spider-nodejs",
   "os": [
     "win32"
diff --git a/npm/win32-x64-msvc/package.json b/npm/win32-x64-msvc/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@spider-rs/spider-rs-win32-x64-msvc",
-  "version": "0.0.87",
+  "version": "0.0.139",
   "repository": "https://github.com/spider-rs/spider-nodejs",
   "os": [
     "win32"
diff --git a/package.json b/package.json
@@ -1,75 +1,78 @@
 {
-        "name": "@spider-rs/spider-rs",
-        "version": "0.0.87",
-        "main": "index.js",
-        "types": "index.d.ts",
-        "napi": {
-                "name": "spider-rs",
-                "triples": {
-                        "additional": [
-                                "aarch64-apple-darwin",
-                                "aarch64-linux-android",
-                                "aarch64-unknown-linux-gnu",
-                                "aarch64-unknown-linux-musl",
-                                "aarch64-pc-windows-msvc",
-                                "armv7-unknown-linux-gnueabihf",
-                                "x86_64-unknown-linux-musl",
-                                "x86_64-unknown-freebsd",
-                                "i686-pc-windows-msvc",
-                                "armv7-linux-androideabi",
-                                "universal-apple-darwin"
-                        ]
-                }
-        },
-        "license": "MIT",
-        "keywords": [
-                "spider",
-                "crawler"
-        ],
-        "repository": "https://github.com/spider-rs/spider-nodejs",
-        "devDependencies": {
-                "@napi-rs/cli": "^2.18.4",
-                "@types/node": "^20.14.5",
-                "ava": "^6.1.3",
-                "typescript": "^5.4.5"
-        },
-        "ava": {
-                "require": [
-                        "@swc-node/register"
-                ],
-                "extensions": [
-                        "ts"
-                ],
-                "timeout": "2m",
-                "workerThreads": false,
-                "environmentVariables": {
-                        "TS_NODE_PROJECT": "./tsconfig.json"
-                }
-        },
-        "engines": {
-                "node": ">= 10"
-        },
-        "scripts": {
-                "artifacts": "napi artifacts",
-                "bench": "cd bench && npm run bench",
-                "bench:oss": "cd bench && npm run bench:oss",
-                "build": "napi build --platform --release --pipe \"prettier -w\"",
-                "build:debug": "napi build --platform --pipe \"prettier -w\"",
-                "format": "run-p format:prettier format:rs format:toml",
-                "format:prettier": "prettier . -w",
-                "format:toml": "taplo format",
-                "format:rs": "cargo fmt",
-                "lint": "eslint . -c ./.eslintrc.yml",
-                "prepublishOnly": "napi prepublish -t npm",
-                "test": "ava",
-                "version": "napi version"
-        },
-        "prettier": {
-                "printWidth": 120,
-                "semi": false,
-                "trailingComma": "all",
-                "singleQuote": true,
-                "arrowParens": "always"
-        },
-        "packageManager": "yarn@4.3.1"
+          "name": "@spider-rs/spider-rs",
+          "version": "0.0.139",
+          "main": "index.js",
+          "types": "index.d.ts",
+          "napi": {
+                    "name": "spider-rs",
+                    "triples": {
+                              "additional": [
+                                        "aarch64-apple-darwin",
+                                        "aarch64-linux-android",
+                                        "aarch64-unknown-linux-gnu",
+                                        "aarch64-unknown-linux-musl",
+                                        "aarch64-pc-windows-msvc",
+                                        "armv7-unknown-linux-gnueabihf",
+                                        "x86_64-unknown-linux-musl",
+                                        "x86_64-unknown-freebsd",
+                                        "i686-pc-windows-msvc",
+                                        "armv7-linux-androideabi",
+                                        "universal-apple-darwin"
+                              ]
+                    }
+          },
+          "license": "MIT",
+          "keywords": [
+                    "spider",
+                    "crawler"
+          ],
+          "repository": "https://github.com/spider-rs/spider-nodejs",
+          "devDependencies": {
+                    "@napi-rs/cli": "^2.18.4",
+                    "@swc-node/register": "^1.10.9",
+                    "@swc/core": "^1.7.0",
+                    "@types/node": "^20.14.5",
+                    "ava": "^6.1.3",
+                    "prettier": "^3.3.3",
+                    "typescript": "^5.4.5"
+          },
+          "ava": {
+                    "require": [
+                              "@swc-node/register"
+                    ],
+                    "extensions": [
+                              "ts"
+                    ],
+                    "timeout": "5m",
+                    "workerThreads": false,
+                    "environmentVariables": {
+                              "TS_NODE_PROJECT": "./tsconfig.json"
+                    }
+          },
+          "engines": {
+                    "node": ">= 10"
+          },
+          "scripts": {
+                    "artifacts": "napi artifacts",
+                    "bench": "cd bench && npm run bench",
+                    "bench:oss": "cd bench && npm run bench:oss",
+                    "build": "napi build --platform --release --pipe \"prettier -w\"",
+                    "build:debug": "napi build --platform --pipe \"prettier -w\"",
+                    "format": "run-p format:prettier format:rs format:toml",
+                    "format:prettier": "prettier . -w",
+                    "format:toml": "taplo format",
+                    "format:rs": "cargo fmt",
+                    "lint": "eslint . -c ./.eslintrc.yml",
+                    "prepublishOnly": "napi prepublish -t npm",
+                    "test": "ava",
+                    "version": "napi version"
+          },
+          "prettier": {
+                    "printWidth": 120,
+                    "semi": false,
+                    "trailingComma": "all",
+                    "singleQuote": true,
+                    "arrowParens": "always"
+          },
+          "packageManager": "yarn@4.3.1"
 }
diff --git a/src/website.rs b/src/website.rs
@@ -686,6 +686,13 @@ impl Website {
     self
   }
 
+  /// Determine whether to collect all the resources found on pages.
+  #[napi]
+  pub fn with_full_resources(&mut self, full_resources: bool) -> &Self {
+    self.inner.configuration.with_full_resources(full_resources);
+    self
+  }
+
   /// Use network interception for the request to only allow content that matches the host. If the content is from a 3rd party it needs to be part of our include list.
   #[napi]
   pub fn with_chrome_intercept(&mut self, chrome_intercept: bool, block_images: bool) -> &Self {
diff --git a/yarn.lock b/yarn.lock

Original file line number	Diff line number	Diff line change
`@@ -1,6 +1,6 @@`
`1`	`1`	`{`
`2`	`2`	`"name": "@spider-rs/spider-rs-android-arm-eabi",`
`3`		`- "version": "0.0.87",`
	`3`	`+ "version": "0.0.139",`
`4`	`4`	`"repository": "https://github.com/spider-rs/spider-nodejs",`
`5`	`5`	`"os": [`
`6`	`6`	`"android"`
Original file line number	Diff line number	Diff line change
`@@ -1,6 +1,6 @@`
`1`	`1`	`{`
`2`	`2`	`"name": "@spider-rs/spider-rs-darwin-arm64",`
`3`		`- "version": "0.0.87",`
	`3`	`+ "version": "0.0.139",`
`4`	`4`	`"repository": "https://github.com/spider-rs/spider-nodejs",`
`5`	`5`	`"os": [`
`6`	`6`	`"darwin"`
Original file line number	Diff line number	Diff line change
`@@ -1,6 +1,6 @@`
`1`	`1`	`{`
`2`	`2`	`"name": "@spider-rs/spider-rs-darwin-universal",`
`3`		`- "version": "0.0.87",`
	`3`	`+ "version": "0.0.139",`
`4`	`4`	`"repository": "https://github.com/spider-rs/spider-nodejs",`
`5`	`5`	`"os": [`
`6`	`6`	`"darwin"`
Original file line number	Diff line number	Diff line change
`@@ -1,6 +1,6 @@`
`1`	`1`	`{`
`2`	`2`	`"name": "@spider-rs/spider-rs-darwin-x64",`
`3`		`- "version": "0.0.87",`
	`3`	`+ "version": "0.0.139",`
`4`	`4`	`"repository": "https://github.com/spider-rs/spider-nodejs",`
`5`	`5`	`"os": [`
`6`	`6`	`"darwin"`
Original file line number	Diff line number	Diff line change
`@@ -1,6 +1,6 @@`
`1`	`1`	`{`
`2`	`2`	`"name": "@spider-rs/spider-rs-freebsd-x64",`
`3`		`- "version": "0.0.87",`
	`3`	`+ "version": "0.0.139",`
`4`	`4`	`"repository": "https://github.com/spider-rs/spider-nodejs",`
`5`	`5`	`"os": [`
`6`	`6`	`"freebsd"`
Original file line number	Diff line number	Diff line change
`@@ -1,6 +1,6 @@`
`1`	`1`	`{`
`2`	`2`	`"name": "@spider-rs/spider-rs-linux-arm-gnueabihf",`
`3`		`- "version": "0.0.87",`
	`3`	`+ "version": "0.0.139",`
`4`	`4`	`"repository": "https://github.com/spider-rs/spider-nodejs",`
`5`	`5`	`"os": [`
`6`	`6`	`"linux"`