Deeplake

DeepLakeVectorStore

Bases: BasePydanticVectorStore

The DeepLake Vector Store.

In this vector store we store the text, its embedding and a few pieces of its metadata in a deeplake dataset. This implementation allows the use of an already existing deeplake dataset if it is one that was created this vector store. It also supports creating a new one if the dataset doesn't exist or if overwrite is set to True.

Examples:

pip install llama-index-vector-stores-deeplake

from llama_index.vector_stores.deeplake import DeepLakeVectorStore

# Create an instance of DeepLakeVectorStore
vector_store = DeepLakeVectorStore(dataset_path=dataset_path, overwrite=True)

Source code in llama-index-integrations/vector_stores/llama-index-vector-stores-deeplake/llama_index/vector_stores/deeplake/base.py

class DeepLakeVectorStore(BasePydanticVectorStore):
    """The DeepLake Vector Store.

    In this vector store we store the text, its embedding and
    a few pieces of its metadata in a deeplake dataset. This implementation
    allows the use of an already existing deeplake dataset if it is one that was created
    this vector store. It also supports creating a new one if the dataset doesn't
    exist or if `overwrite` is set to True.

    Examples:
        `pip install llama-index-vector-stores-deeplake`

        ```python
        from llama_index.vector_stores.deeplake import DeepLakeVectorStore

        # Create an instance of DeepLakeVectorStore
        vector_store = DeepLakeVectorStore(dataset_path=dataset_path, overwrite=True)
        ```
    """

    stores_text: bool = True
    flat_metadata: bool = True

    ingestion_batch_size: int
    num_workers: int
    token: Optional[str]
    read_only: Optional[bool]
    dataset_path: str
    vectorstore: Any = "VectorStore"

    _embedding_dimension: int = PrivateAttr()
    _ttl_seconds: Optional[int] = PrivateAttr()
    _deeplake_db: Any = PrivateAttr()
    _deeplake_db_collection: Any = PrivateAttr()
    _id_tensor_name: str = PrivateAttr()

    def __init__(
        self,
        dataset_path: str = "llama_index",
        token: Optional[str] = None,
        read_only: Optional[bool] = False,
        ingestion_batch_size: int = 1024,
        ingestion_num_workers: int = 4,
        overwrite: bool = False,
        exec_option: Optional[str] = None,
        verbose: bool = True,
        **kwargs: Any,
    ) -> None:
        """
        Args:
            dataset_path (str): The full path for storing to the Deep Lake Vector Store. It can be:
                - a Deep Lake cloud path of the form ``hub://org_id/dataset_name``. Requires registration with Deep Lake.
                - an s3 path of the form ``s3://bucketname/path/to/dataset``. Credentials are required in either the environment or passed to the creds argument.
                - a local file system path of the form ``./path/to/dataset`` or ``~/path/to/dataset`` or ``path/to/dataset``.
                - a memory path of the form ``mem://path/to/dataset`` which doesn't save the dataset but keeps it in memory instead. Should be used only for testing as it does not persist.
                Defaults to "llama_index".
            overwrite (bool, optional): If set to True this overwrites the Vector Store if it already exists. Defaults to False.
            token (str, optional): Activeloop token, used for fetching user credentials. This is Optional, tokens are normally autogenerated. Defaults to None.
            read_only (bool, optional): Opens dataset in read-only mode if True. Defaults to False.
            ingestion_batch_size (int): During data ingestion, data is divided
                into batches. Batch size is the size of each batch. Defaults to 1024.
            ingestion_num_workers (int): number of workers to use during data ingestion.
                Defaults to 4.
            exec_option (str): Default method for search execution. It could be either ``"auto"``, ``"python"``, ``"compute_engine"`` or ``"tensor_db"``. Defaults to ``"auto"``. If None, it's set to "auto".
                - ``auto``- Selects the best execution method based on the storage location of the Vector Store. It is the default option.
                - ``python`` - Pure-python implementation that runs on the client and can be used for data stored anywhere. WARNING: using this option with big datasets is discouraged because it can lead to memory issues.
                - ``compute_engine`` - Performant C++ implementation of the Deep Lake Compute Engine that runs on the client and can be used for any data stored in or connected to Deep Lake. It cannot be used with in-memory or local datasets.
                - ``tensor_db`` - Performant and fully-hosted Managed Tensor Database that is responsible for storage and query execution. Only available for data stored in the Deep Lake Managed Database. Store datasets in this database by specifying runtime = {"tensor_db": True} during dataset creation.

        Raises:
            ImportError: Unable to import `deeplake`.
        """
        super().__init__(
            dataset_path=dataset_path,
            token=token,
            read_only=read_only,
            ingestion_batch_size=ingestion_batch_size,
            num_workers=ingestion_num_workers,
        )

        self.vectorstore = VectorStore(
            path=dataset_path,
            ingestion_batch_size=ingestion_batch_size,
            num_workers=ingestion_num_workers,
            token=token,
            read_only=read_only,
            exec_option=exec_option,
            overwrite=overwrite,
            verbose=verbose,
            **kwargs,
        )
        try:
            self._id_tensor_name = (
                "ids" if "ids" in self.vectorstore.tensors() else "id"
            )
        except AttributeError:
            self._id_tensor_name = "id"

    @property
    def client(self) -> Any:
        """Get client.

        Returns:
            Any: DeepLake vectorstore dataset.
        """
        return self.vectorstore.dataset

    def summary(self):
        self.vectorstore.summary()

    def get_nodes(
        self,
        node_ids: Optional[List[str]] = None,
        filters: Optional[MetadataFilters] = None,
    ) -> List[BaseNode]:
        """Get nodes from vector store."""
        if node_ids:
            data = self.vectorstore.search(filter={"id": node_ids})
        else:
            data = self.vectorstore.search(filter={})

        nodes = []
        for metadata in data["metadata"]:
            nodes.append(metadata_dict_to_node(metadata))

        def filter_func(doc):
            if not filters:
                return True

            found_one = False
            for f in filters.filters:
                value = doc.metadata[f.key]
                if f.operator == FilterOperator.EQ:
                    result = value == f.value
                elif f.operator == FilterOperator.GT:
                    result = value > f.value
                elif f.operator == FilterOperator.GTE:
                    result = value >= f.value
                elif f.operator == FilterOperator.LT:
                    result = value < f.value
                elif f.operator == FilterOperator.LTE:
                    result = value <= f.value
                elif f.operator == FilterOperator.NE:
                    result = value != f.value
                elif f.operator == FilterOperator.IN:
                    result = value in f.value
                elif f.operator == FilterOperator.NOT_IN:
                    result = value not in f.value
                elif f.operator == FilterOperator.TEXT_MATCH:
                    result = f.value in value
                else:
                    raise ValueError(f"Unsupported filter operator: {f.operator}")

                if result:
                    found_one = True
                    if filters.condition == FilterCondition.OR:
                        return True
                else:
                    if filters.condition == FilterCondition.AND:
                        return False

            return found_one

        if filters:
            return [x for x in nodes if filter_func(x)]
        else:
            return nodes

    def delete_nodes(
        self,
        node_ids: Optional[List[str]] = None,
        filters: Optional[MetadataFilters] = None,
        **delete_kwargs: Any,
    ) -> None:
        if filters:
            self.vectorstore.delete(
                ids=[
                    x.node_id
                    for x in self.get_nodes(node_ids=node_ids, filters=filters)
                ]
            )
        else:
            self.vectorstore.delete(ids=node_ids)

    def clear(self) -> None:
        """Clear the vector store."""
        if DEEPLAKE_V4:
            for i in range(len(self.vectorstore.ds) - 1, -1, -1):
                self.vectorstore.ds.delete(i)
        else:
            self.vectorstore.delete(filter=lambda x: True)

    def add(self, nodes: List[BaseNode], **add_kwargs: Any) -> List[str]:
        """Add the embeddings and their nodes into DeepLake.

        Args:
            nodes (List[BaseNode]): List of nodes with embeddings
                to insert.

        Returns:
            List[str]: List of ids inserted.
        """
        embedding = []
        metadata = []
        id_ = []
        text = []

        for node in nodes:
            embedding.append(node.get_embedding())
            metadata.append(
                node_to_metadata_dict(
                    node, remove_text=False, flat_metadata=self.flat_metadata
                )
            )
            id_.append(node.node_id)
            text.append(node.get_content(metadata_mode=MetadataMode.NONE))

        if DEEPLAKE_V4:
            kwargs = {self._id_tensor_name: id_}

            return self.vectorstore.add(
                embedding_data=embedding,
                embedding_tensor="embedding",
                metadata=metadata,
                text=text,
                return_ids=True,
                **kwargs,
            )
        else:
            kwargs = {
                "embedding": embedding,
                "metadata": metadata,
                self._id_tensor_name: id_,
                "text": text,
            }
            return self.vectorstore.add(
                return_ids=True,
                **kwargs,
            )

    def delete(self, ref_doc_id: str, **delete_kwargs: Any) -> None:
        """
        Delete nodes using with ref_doc_id.

        Args:
            ref_doc_id (str): The doc_id of the document to delete.

        """
        self.vectorstore.delete(filter={"metadata": {"doc_id": ref_doc_id}})

    def query(self, query: VectorStoreQuery, **kwargs: Any) -> VectorStoreQueryResult:
        """Query index for top k most similar nodes.

        Args:
            query (VectorStoreQuery): VectorStoreQuery class input, it has
                the following attributes:
                1. query_embedding (List[float]): query embedding
                2. similarity_top_k (int): top k most similar nodes
            deep_memory (bool): Whether to use deep memory for query execution.

        Returns:
            VectorStoreQueryResult
        """
        query_embedding = cast(List[float], query.query_embedding)
        exec_option = kwargs.get("exec_option")
        deep_memory = kwargs.get("deep_memory")
        data = self.vectorstore.search(
            embedding=query_embedding,
            exec_option=exec_option,
            k=query.similarity_top_k,
            distance_metric="cos",
            filter=query.filters,
            return_tensors=None,
            deep_memory=deep_memory,
        )

        similarities = data["score"]
        ids = data[self._id_tensor_name]
        metadatas = data["metadata"]
        nodes = []
        for metadata in metadatas:
            if "_node_type" not in metadata:
                metadata["_node_type"] = TextNode.class_name()
            nodes.append(metadata_dict_to_node(metadata))

        return VectorStoreQueryResult(nodes=nodes, similarities=similarities, ids=ids)

client property

client: Any

Get client.

Returns:

Name	Type	Description
Any	Any	DeepLake vectorstore dataset.

get_nodes

get_nodes(node_ids: Optional[List[str]] = None, filters: Optional[MetadataFilters] = None) -> List[BaseNode]

Get nodes from vector store.

Source code in llama-index-integrations/vector_stores/llama-index-vector-stores-deeplake/llama_index/vector_stores/deeplake/base.py

def get_nodes(
    self,
    node_ids: Optional[List[str]] = None,
    filters: Optional[MetadataFilters] = None,
) -> List[BaseNode]:
    """Get nodes from vector store."""
    if node_ids:
        data = self.vectorstore.search(filter={"id": node_ids})
    else:
        data = self.vectorstore.search(filter={})

    nodes = []
    for metadata in data["metadata"]:
        nodes.append(metadata_dict_to_node(metadata))

    def filter_func(doc):
        if not filters:
            return True

        found_one = False
        for f in filters.filters:
            value = doc.metadata[f.key]
            if f.operator == FilterOperator.EQ:
                result = value == f.value
            elif f.operator == FilterOperator.GT:
                result = value > f.value
            elif f.operator == FilterOperator.GTE:
                result = value >= f.value
            elif f.operator == FilterOperator.LT:
                result = value < f.value
            elif f.operator == FilterOperator.LTE:
                result = value <= f.value
            elif f.operator == FilterOperator.NE:
                result = value != f.value
            elif f.operator == FilterOperator.IN:
                result = value in f.value
            elif f.operator == FilterOperator.NOT_IN:
                result = value not in f.value
            elif f.operator == FilterOperator.TEXT_MATCH:
                result = f.value in value
            else:
                raise ValueError(f"Unsupported filter operator: {f.operator}")

            if result:
                found_one = True
                if filters.condition == FilterCondition.OR:
                    return True
            else:
                if filters.condition == FilterCondition.AND:
                    return False

        return found_one

    if filters:
        return [x for x in nodes if filter_func(x)]
    else:
        return nodes

clear

clear() -> None

Clear the vector store.

Source code in llama-index-integrations/vector_stores/llama-index-vector-stores-deeplake/llama_index/vector_stores/deeplake/base.py

def clear(self) -> None:
    """Clear the vector store."""
    if DEEPLAKE_V4:
        for i in range(len(self.vectorstore.ds) - 1, -1, -1):
            self.vectorstore.ds.delete(i)
    else:
        self.vectorstore.delete(filter=lambda x: True)

add

add(nodes: List[BaseNode], **add_kwargs: Any) -> List[str]

Add the embeddings and their nodes into DeepLake.

Parameters:

Name	Type	Description	Default
nodes	List[BaseNode]	List of nodes with embeddings to insert.	required

Returns:

Type	Description
List[str]	List[str]: List of ids inserted.

Source code in llama-index-integrations/vector_stores/llama-index-vector-stores-deeplake/llama_index/vector_stores/deeplake/base.py

def add(self, nodes: List[BaseNode], **add_kwargs: Any) -> List[str]:
    """Add the embeddings and their nodes into DeepLake.

    Args:
        nodes (List[BaseNode]): List of nodes with embeddings
            to insert.

    Returns:
        List[str]: List of ids inserted.
    """
    embedding = []
    metadata = []
    id_ = []
    text = []

    for node in nodes:
        embedding.append(node.get_embedding())
        metadata.append(
            node_to_metadata_dict(
                node, remove_text=False, flat_metadata=self.flat_metadata
            )
        )
        id_.append(node.node_id)
        text.append(node.get_content(metadata_mode=MetadataMode.NONE))

    if DEEPLAKE_V4:
        kwargs = {self._id_tensor_name: id_}

        return self.vectorstore.add(
            embedding_data=embedding,
            embedding_tensor="embedding",
            metadata=metadata,
            text=text,
            return_ids=True,
            **kwargs,
        )
    else:
        kwargs = {
            "embedding": embedding,
            "metadata": metadata,
            self._id_tensor_name: id_,
            "text": text,
        }
        return self.vectorstore.add(
            return_ids=True,
            **kwargs,
        )

delete

delete(ref_doc_id: str, **delete_kwargs: Any) -> None

Delete nodes using with ref_doc_id.

Parameters:

Name	Type	Description	Default
ref_doc_id	str	The doc_id of the document to delete.	required

Source code in llama-index-integrations/vector_stores/llama-index-vector-stores-deeplake/llama_index/vector_stores/deeplake/base.py

def delete(self, ref_doc_id: str, **delete_kwargs: Any) -> None:
    """
    Delete nodes using with ref_doc_id.

    Args:
        ref_doc_id (str): The doc_id of the document to delete.

    """
    self.vectorstore.delete(filter={"metadata": {"doc_id": ref_doc_id}})

query

query(query: VectorStoreQuery, **kwargs: Any) -> VectorStoreQueryResult

Query index for top k most similar nodes.

Parameters:

Name	Type	Description	Default
query	VectorStoreQuery	VectorStoreQuery class input, it has the following attributes: 1. query_embedding (List[float]): query embedding 2. similarity_top_k (int): top k most similar nodes	required
deep_memory	bool	Whether to use deep memory for query execution.	required

Returns:

Type	Description
VectorStoreQueryResult	VectorStoreQueryResult

Source code in llama-index-integrations/vector_stores/llama-index-vector-stores-deeplake/llama_index/vector_stores/deeplake/base.py

def query(self, query: VectorStoreQuery, **kwargs: Any) -> VectorStoreQueryResult:
    """Query index for top k most similar nodes.

    Args:
        query (VectorStoreQuery): VectorStoreQuery class input, it has
            the following attributes:
            1. query_embedding (List[float]): query embedding
            2. similarity_top_k (int): top k most similar nodes
        deep_memory (bool): Whether to use deep memory for query execution.

    Returns:
        VectorStoreQueryResult
    """
    query_embedding = cast(List[float], query.query_embedding)
    exec_option = kwargs.get("exec_option")
    deep_memory = kwargs.get("deep_memory")
    data = self.vectorstore.search(
        embedding=query_embedding,
        exec_option=exec_option,
        k=query.similarity_top_k,
        distance_metric="cos",
        filter=query.filters,
        return_tensors=None,
        deep_memory=deep_memory,
    )

    similarities = data["score"]
    ids = data[self._id_tensor_name]
    metadatas = data["metadata"]
    nodes = []
    for metadata in metadatas:
        if "_node_type" not in metadata:
            metadata["_node_type"] = TextNode.class_name()
        nodes.append(metadata_dict_to_node(metadata))

    return VectorStoreQueryResult(nodes=nodes, similarities=similarities, ids=ids)