fix: Removed link_urls and img_ulrs from FetchNode output

LorenzoPaleari · LorenzoPaleari · commit 57337a0a8c86 · 2024-09-02T12:53:40.000+02:00
diff --git a/examples/anthropic/custom_graph_haiku.py b/examples/anthropic/custom_graph_haiku.py
@@ -40,7 +40,7 @@
 
 fetch_node = FetchNode(
     input="url | local_dir",
-    output=["doc", "link_urls", "img_urls"],
+    output=["doc"],
     node_config={
         "verbose": True,
         "headless": True,
diff --git a/examples/bedrock/custom_graph_bedrock.py b/examples/bedrock/custom_graph_bedrock.py
@@ -55,7 +55,7 @@
 
 fetch_node = FetchNode(
     input="url | local_dir",
-    output=["doc", "link_urls", "img_urls"],
+    output=["doc"],
     node_config={
         "verbose": True,
         "headless": True,
diff --git a/examples/ernie/custom_graph_ernie.py b/examples/ernie/custom_graph_ernie.py
@@ -43,7 +43,7 @@
 
 fetch_node = FetchNode(
     input="url | local_dir",
-    output=["doc", "link_urls", "img_urls"],
+    output=["doc"],
     node_config={
         "verbose": True,
         "headless": True,
diff --git a/examples/fireworks/custom_graph_fireworks.py b/examples/fireworks/custom_graph_fireworks.py
@@ -43,7 +43,7 @@
 
 fetch_node = FetchNode(
     input="url | local_dir",
-    output=["doc", "link_urls", "img_urls"],
+    output=["doc"],
     node_config={
         "verbose": True,
         "headless": True,
diff --git a/examples/groq/custom_graph_groq.py b/examples/groq/custom_graph_groq.py
@@ -43,7 +43,7 @@
 
 fetch_node = FetchNode(
     input="url | local_dir",
-    output=["doc", "link_urls", "img_urls"],
+    output=["doc"],
     node_config={
         "verbose": True,
         "headless": True,
diff --git a/examples/huggingfacehub/custom_graph_huggingfacehub.py b/examples/huggingfacehub/custom_graph_huggingfacehub.py
@@ -55,7 +55,7 @@
 
 fetch_node = FetchNode(
     input="url | local_dir",
-    output=["doc", "link_urls", "img_urls"],
+    output=["doc"],
     node_config={
         "verbose": True,
         "headless": True,
diff --git a/examples/local_models/custom_graph_ollama.py b/examples/local_models/custom_graph_ollama.py
@@ -44,7 +44,7 @@
 
 fetch_node = FetchNode(
     input="url | local_dir",
-    output=["doc", "link_urls", "img_urls"],
+    output=["doc"],
     node_config={
         "verbose": True,
         "headless": True,
diff --git a/examples/mistral/custom_graph_mistral.py b/examples/mistral/custom_graph_mistral.py
@@ -42,7 +42,7 @@
 
 fetch_node = FetchNode(
     input="url | local_dir",
-    output=["doc", "link_urls", "img_urls"],
+    output=["doc"],
     node_config={
         "verbose": True,
         "headless": True,
diff --git a/examples/mixed_models/custom_graph_groq_openai.py b/examples/mixed_models/custom_graph_groq_openai.py
@@ -51,7 +51,7 @@
 
 fetch_node = FetchNode(
     input="url | local_dir",
-    output=["doc", "link_urls", "img_urls"],
+    output=["doc"],
     node_config={
         "verbose": True,
         "headless": True,
diff --git a/examples/nemotron/custom_graph_nemotron.py b/examples/nemotron/custom_graph_nemotron.py
@@ -42,7 +42,7 @@
 
 fetch_node = FetchNode(
     input="url | local_dir",
-    output=["doc", "link_urls", "img_urls"],
+    output=["doc"],
     node_config={
         "verbose": True,
         "headless": True,
diff --git a/examples/oneapi/custom_graph_oneapi.py b/examples/oneapi/custom_graph_oneapi.py
@@ -38,7 +38,7 @@
 
 fetch_node = FetchNode(
     input="url | local_dir",
-    output=["doc", "link_urls", "img_urls"],
+    output=["doc"],
     node_config={
         "verbose": True,
         "headless": True,
diff --git a/examples/openai/custom_graph_openai.py b/examples/openai/custom_graph_openai.py
@@ -43,7 +43,7 @@
 
 fetch_node = FetchNode(
     input="url | local_dir",
-    output=["doc", "link_urls", "img_urls"],
+    output=["doc"],
     node_config={
         "verbose": True,
         "headless": True,
diff --git a/scrapegraphai/graphs/deep_scraper_graph.py b/scrapegraphai/graphs/deep_scraper_graph.py
@@ -69,7 +69,7 @@ def _create_repeated_graph(self) -> BaseGraph:
         """
         fetch_node = FetchNode(
             input="url | local_dir",
-            output=["doc", "link_urls", "img_urls"]
+            output=["doc"]
         )
         parse_node = ParseNode(
             input="doc",
diff --git a/scrapegraphai/graphs/json_scraper_graph.py b/scrapegraphai/graphs/json_scraper_graph.py
@@ -56,7 +56,7 @@ def _create_graph(self) -> BaseGraph:
 
         fetch_node = FetchNode(
             input="json | json_dir",
-            output=["doc", "link_urls", "img_urls"],
+            output=["doc"],
         )
 
         generate_answer_node = GenerateAnswerNode(
diff --git a/scrapegraphai/graphs/script_creator_graph.py b/scrapegraphai/graphs/script_creator_graph.py
@@ -62,7 +62,7 @@ def _create_graph(self) -> BaseGraph:
 
         fetch_node = FetchNode(
             input="url | local_dir",
-            output=["doc", "link_urls", "img_urls"],
+            output=["doc"],
             node_config={
                 "llm_model": self.llm_model,
                 "loader_kwargs": self.config.get("loader_kwargs", {}),
diff --git a/scrapegraphai/graphs/search_link_graph.py b/scrapegraphai/graphs/search_link_graph.py
@@ -52,7 +52,7 @@ def _create_graph(self) -> BaseGraph:
 
         fetch_node = FetchNode(
             input="url| local_dir",
-            output=["doc", "link_urls", "img_urls"],
+            output=["doc"],
             node_config={
                 "llm_model": self.llm_model,
                 "force": self.config.get("force", False),
diff --git a/scrapegraphai/graphs/smart_scraper_graph.py b/scrapegraphai/graphs/smart_scraper_graph.py
@@ -61,7 +61,7 @@ def _create_graph(self) -> BaseGraph:
         """
         fetch_node = FetchNode(
             input="url| local_dir",
-            output=["doc", "link_urls", "img_urls"],
+            output=["doc"],
             node_config={
                 "llm_model": self.llm_model,
                 "force": self.config.get("force", False),
diff --git a/scrapegraphai/graphs/speech_graph.py b/scrapegraphai/graphs/speech_graph.py
@@ -62,7 +62,7 @@ def _create_graph(self) -> BaseGraph:
 
         fetch_node = FetchNode(
             input="url | local_dir",
-            output=["doc", "link_urls", "img_urls"]
+            output=["doc"]
         )
         parse_node = ParseNode(
             input="doc",
diff --git a/scrapegraphai/graphs/xml_scraper_graph.py b/scrapegraphai/graphs/xml_scraper_graph.py
@@ -60,7 +60,7 @@ def _create_graph(self) -> BaseGraph:
 
         fetch_node = FetchNode(
             input="xml | xml_dir",
-            output=["doc", "link_urls", "img_urls"]
+            output=["doc"]
         )
 
         generate_answer_node = GenerateAnswerNode(
diff --git a/tests/graphs/abstract_graph_test.py b/tests/graphs/abstract_graph_test.py
@@ -22,7 +22,7 @@ def __init__(self, prompt: str, config: dict):
     def _create_graph(self) -> BaseGraph:
         fetch_node = FetchNode(
             input="url| local_dir",
-            output=["doc", "link_urls", "img_urls"],
+            output=["doc"],
             node_config={
                 "llm_model": self.llm_model,
                 "force": self.config.get("force", False),

Original file line number	Diff line number	Diff line change
`@@ -69,7 +69,7 @@ def _create_repeated_graph(self) -> BaseGraph:`
`69`	`69`	`"""`
`70`	`70`	`fetch_node = FetchNode(`
`71`	`71`	`input="url \| local_dir",`
`72`		`- output=["doc", "link_urls", "img_urls"]`
	`72`	`+ output=["doc"]`
`73`	`73`	`)`
`74`	`74`	`parse_node = ParseNode(`
`75`	`75`	`input="doc",`
Original file line number	Diff line number	Diff line change
`@@ -56,7 +56,7 @@ def _create_graph(self) -> BaseGraph:`
`56`	`56`
`57`	`57`	`fetch_node = FetchNode(`
`58`	`58`	`input="json \| json_dir",`
`59`		`- output=["doc", "link_urls", "img_urls"],`
	`59`	`+ output=["doc"],`
`60`	`60`	`)`
`61`	`61`
`62`	`62`	`generate_answer_node = GenerateAnswerNode(`
Original file line number	Diff line number	Diff line change
`@@ -62,7 +62,7 @@ def _create_graph(self) -> BaseGraph:`
`62`	`62`
`63`	`63`	`fetch_node = FetchNode(`
`64`	`64`	`input="url \| local_dir",`
`65`		`- output=["doc", "link_urls", "img_urls"]`
	`65`	`+ output=["doc"]`
`66`	`66`	`)`
`67`	`67`	`parse_node = ParseNode(`
`68`	`68`	`input="doc",`
Original file line number	Diff line number	Diff line change
`@@ -60,7 +60,7 @@ def _create_graph(self) -> BaseGraph:`
`60`	`60`
`61`	`61`	`fetch_node = FetchNode(`
`62`	`62`	`input="xml \| xml_dir",`
`63`		`- output=["doc", "link_urls", "img_urls"]`
	`63`	`+ output=["doc"]`
`64`	`64`	`)`
`65`	`65`
`66`	`66`	`generate_answer_node = GenerateAnswerNode(`