Remove jq as a dependency and adopt pip-tools to manage all dependencies (elastic#122)

Author: miguelgrinberg

example-apps/chatbot-rag-app/data/index_data.py

@@ -1,8 +1,9 @@
 from elasticsearch import Elasticsearch, NotFoundError
 from langchain.vectorstores import ElasticsearchStore
-from langchain.document_loaders import JSONLoader
+from langchain.docstore.document import Document
 from langchain.text_splitter import RecursiveCharacterTextSplitter
 from dotenv import load_dotenv
+import json
 import os
 import time
 
@@ -55,29 +56,19 @@ def install_elser():
         )
 
 
-# Metadata extraction function
-def metadata_func(record: dict, metadata: dict) -> dict:
-    metadata["name"] = record.get("name")
-    metadata["summary"] = record.get("summary")
-    metadata["url"] = record.get("url")
-    metadata["category"] = record.get("category")
-    metadata["updated_at"] = record.get("updated_at")
-
-    return metadata
-
-
 def main():
     install_elser()
 
     print(f"Loading data from ${FILE}")
 
-    loader = JSONLoader(
-        file_path=FILE,
-        jq_schema=".[]",
-        content_key="content",
-        metadata_func=metadata_func,
-    )
-    workplace_docs = loader.load()
+    metadata_keys = ['name', 'summary', 'url', 'category', 'updated_at']
+    workplace_docs = []
+    with open(FILE, 'rt') as f:
+        for doc in json.loads(f.read()):
+            workplace_docs.append(Document(
+                page_content=doc['content'],
+                metadata={k: doc.get(k) for k in metadata_keys}
+            ))
 
     print(f"Loaded {len(workplace_docs)} documents")
 

example-apps/chatbot-rag-app/requirements.in

@@ -0,0 +1,25 @@
+# package management
+pip-tools
+
+# core dependencies
+elasticsearch
+langchain
+tiktoken
+flask
+flask-cors
+python-dotenv
+
+# OpenAI dependencies
+openai
+
+# Vertex AI dependencies
+google-cloud-aiplatform
+grpcio-status
+
+# BedRock dependencies
+boto3
+
+# TBD if these are still needed
+exceptiongroup
+importlib-metadata
+numexpr

example-apps/chatbot-rag-app/requirements.txt

@@ -1,76 +1,259 @@
+#
+# This file is autogenerated by pip-compile with Python 3.11
+# by the following command:
+#
+#    pip-compile requirements.in
+#
 aiohttp==3.8.5
+    # via
+    #   langchain
+    #   openai
 aiosignal==1.3.1
+    # via aiohttp
 annotated-types==0.5.0
+    # via pydantic
 anyio==3.7.1
+    # via langchain
 async-timeout==4.0.3
+    # via aiohttp
 attrs==23.1.0
+    # via aiohttp
 blinker==1.6.2
+    # via flask
 boto3==1.28.61
+    # via -r requirements.in
 botocore==1.31.61
+    # via
+    #   boto3
+    #   s3transfer
+build==1.0.3
+    # via
+    #   -r requirements.in
+    #   pip-tools
 cachetools==5.3.1
+    # via google-auth
 certifi==2023.7.22
+    # via
+    #   elastic-transport
+    #   requests
 charset-normalizer==3.2.0
+    # via
+    #   aiohttp
+    #   requests
 click==8.1.7
+    # via
+    #   flask
+    #   pip-tools
 dataclasses-json==0.5.14
+    # via langchain
 elastic-transport==8.4.0
+    # via elasticsearch
 elasticsearch==8.9.0
-exceptiongroup==1.1.3
-Flask==2.3.3
-Flask-Cors==4.0.0
+    # via -r requirements.in
+exceptiongroup==1.2.0
+    # via -r requirements.in
+flask==2.3.3
+    # via
+    #   -r requirements.in
+    #   flask-cors
+flask-cors==4.0.0
+    # via -r requirements.in
 frozenlist==1.4.0
-google-api-core==2.14.0
+    # via
+    #   aiohttp
+    #   aiosignal
+google-api-core[grpc]==2.14.0
+    # via
+    #   google-cloud-aiplatform
+    #   google-cloud-bigquery
+    #   google-cloud-core
+    #   google-cloud-resource-manager
+    #   google-cloud-storage
 google-auth==2.23.2
+    # via
+    #   google-api-core
+    #   google-cloud-core
+    #   google-cloud-storage
 google-cloud-aiplatform==1.35.0
+    # via -r requirements.in
 google-cloud-bigquery==3.13.0
+    # via google-cloud-aiplatform
 google-cloud-core==2.3.3
+    # via
+    #   google-cloud-bigquery
+    #   google-cloud-storage
 google-cloud-resource-manager==1.10.4
+    # via google-cloud-aiplatform
 google-cloud-storage==2.11.0
+    # via google-cloud-aiplatform
 google-crc32c==1.5.0
+    # via google-resumable-media
 google-resumable-media==2.6.0
-googleapis-common-protos==1.61.0
+    # via
+    #   google-cloud-bigquery
+    #   google-cloud-storage
+googleapis-common-protos[grpc]==1.61.0
+    # via
+    #   google-api-core
+    #   grpc-google-iam-v1
+    #   grpcio-status
 grpc-google-iam-v1==0.12.7
+    # via google-cloud-resource-manager
 grpcio==1.59.3
+    # via
+    #   google-api-core
+    #   google-cloud-bigquery
+    #   googleapis-common-protos
+    #   grpc-google-iam-v1
+    #   grpcio-status
 grpcio-status==1.59.3
+    # via
+    #   -r requirements.in
+    #   google-api-core
 idna==3.4
+    # via
+    #   anyio
+    #   requests
+    #   yarl
 importlib-metadata==6.8.0
+    # via -r requirements.in
 itsdangerous==2.1.2
-Jinja2==3.1.2
+    # via flask
+jinja2==3.1.2
+    # via flask
 jmespath==1.0.1
-jq==1.4.1
+    # via
+    #   boto3
+    #   botocore
 jsonpatch==1.33
+    # via langchain
 jsonpointer==2.4
+    # via jsonpatch
 langchain==0.0.333
+    # via -r requirements.in
 langsmith==0.0.65
-MarkupSafe==2.1.3
+    # via langchain
+markupsafe==2.1.3
+    # via
+    #   jinja2
+    #   werkzeug
 marshmallow==3.20.1
+    # via dataclasses-json
 multidict==6.0.4
+    # via
+    #   aiohttp
+    #   yarl
 mypy-extensions==1.0.0
+    # via typing-inspect
 numexpr==2.8.5
+    # via -r requirements.in
 numpy==1.25.2
+    # via
+    #   langchain
+    #   numexpr
+    #   shapely
 openai==0.27.9
+    # via -r requirements.in
 packaging==23.1
+    # via
+    #   build
+    #   google-cloud-aiplatform
+    #   google-cloud-bigquery
+    #   marshmallow
+pip-tools==7.3.0
+    # via -r requirements.in
 proto-plus==1.22.3
+    # via
+    #   google-cloud-aiplatform
+    #   google-cloud-bigquery
+    #   google-cloud-resource-manager
 protobuf==4.25.1
+    # via
+    #   google-api-core
+    #   google-cloud-aiplatform
+    #   google-cloud-bigquery
+    #   google-cloud-resource-manager
+    #   googleapis-common-protos
+    #   grpc-google-iam-v1
+    #   grpcio-status
+    #   proto-plus
 pyasn1==0.5.0
+    # via
+    #   pyasn1-modules
+    #   rsa
 pyasn1-modules==0.3.0
+    # via google-auth
 pydantic==2.3.0
-pydantic_core==2.6.3
+    # via
+    #   langchain
+    #   langsmith
+pydantic-core==2.6.3
+    # via pydantic
+pyproject-hooks==1.0.0
+    # via
+    #   -r requirements.in
+    #   build
 python-dateutil==2.8.2
+    # via
+    #   botocore
+    #   google-cloud-bigquery
 python-dotenv==1.0.0
-PyYAML==6.0.1
+    # via -r requirements.in
+pyyaml==6.0.1
+    # via langchain
+regex==2023.10.3
+    # via tiktoken
 requests==2.31.0
+    # via
+    #   google-api-core
+    #   google-cloud-bigquery
+    #   google-cloud-storage
+    #   langchain
+    #   langsmith
+    #   openai
+    #   tiktoken
 rsa==4.9
+    # via google-auth
 s3transfer==0.7.0
+    # via boto3
 shapely==2.0.2
+    # via google-cloud-aiplatform
 six==1.16.0
+    # via python-dateutil
 sniffio==1.3.0
-SQLAlchemy==2.0.20
+    # via anyio
+sqlalchemy==2.0.20
+    # via langchain
 tenacity==8.2.3
+    # via langchain
+tiktoken==0.5.1
+    # via -r requirements.in
 tqdm==4.66.1
+    # via openai
+typing-extensions==4.7.1
+    # via
+    #   pydantic
+    #   pydantic-core
+    #   sqlalchemy
+    #   typing-inspect
 typing-inspect==0.9.0
-typing_extensions==4.7.1
+    # via dataclasses-json
 urllib3==1.26.16
-Werkzeug==2.3.7
+    # via
+    #   botocore
+    #   elastic-transport
+    #   requests
+werkzeug==2.3.7
+    # via flask
+wheel==0.41.3
+    # via pip-tools
 yarl==1.9.2
+    # via aiohttp
 zipp==3.17.0
-tiktoken==0.5.1
+    # via
+    #   -r requirements.in
+    #   importlib-metadata
+
+# The following packages are considered to be unsafe in a requirements file:
+# pip
+# setuptools