Skip to content

Notion

NotionPageReader #

Bases: BasePydanticReader

Notion Page reader.

Reads a set of Notion pages.

Parameters:

Name Type Description Default
integration_token str

Notion integration token.

 None
Source code in llama-index-integrations/readers/llama-index-readers-notion/llama_index/readers/notion/base.py 
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
class NotionPageReader(BasePydanticReader):
    """Notion Page reader.

    Reads a set of Notion pages.

    Args:
        integration_token (str): Notion integration token.

    """

    is_remote: bool = True
    token: str
    headers: Dict[str, str]

    def __init__(self, integration_token: Optional[str] = None) -> None:
        """Initialize with parameters."""
        if integration_token is None:
            integration_token = os.getenv(INTEGRATION_TOKEN_NAME)
            if integration_token is None:
                raise ValueError(
                    "Must specify `integration_token` or set environment "
                    "variable `NOTION_INTEGRATION_TOKEN`."
                )

        token = integration_token
        headers = {
            "Authorization": "Bearer " + token,
            "Content-Type": "application/json",
            "Notion-Version": "2022-06-28",
        }

        super().__init__(token=token, headers=headers)

    @classmethod
    def class_name(cls) -> str:
        """Get the name identifier of the class."""
        return "NotionPageReader"

    def _read_block(self, block_id: str, num_tabs: int = 0) -> str:
        """Read a block."""
        done = False
        result_lines_arr = []
        cur_block_id = block_id
        while not done:
            block_url = BLOCK_CHILD_URL_TMPL.format(block_id=cur_block_id)
            query_dict: Dict[str, Any] = {}

            res = requests.request(
                "GET", block_url, headers=self.headers, json=query_dict
            )
            data = res.json()

            for result in data["results"]:
                result_type = result["type"]
                result_obj = result[result_type]

                cur_result_text_arr = []
                if "rich_text" in result_obj:
                    for rich_text in result_obj["rich_text"]:
                        # skip if doesn't have text object
                        if "text" in rich_text:
                            text = rich_text["text"]["content"]
                            prefix = "\t" * num_tabs
                            cur_result_text_arr.append(prefix + text)

                result_block_id = result["id"]
                has_children = result["has_children"]
                if has_children:
                    children_text = self._read_block(
                        result_block_id, num_tabs=num_tabs + 1
                    )
                    cur_result_text_arr.append(children_text)

                cur_result_text = "\n".join(cur_result_text_arr)
                result_lines_arr.append(cur_result_text)

            if data["next_cursor"] is None:
                done = True
                break
            else:
                cur_block_id = data["next_cursor"]

        return "\n".join(result_lines_arr)

    def read_page(self, page_id: str) -> str:
        """Read a page."""
        return self._read_block(page_id)

    def query_database(
        self, database_id: str, query_dict: Dict[str, Any] = {"page_size": 100}
    ) -> List[str]:
        """Get all the pages from a Notion database."""
        pages = []

        res = requests.post(
            DATABASE_URL_TMPL.format(database_id=database_id),
            headers=self.headers,
            json=query_dict,
        )
        res.raise_for_status()
        data = res.json()

        pages.extend(data.get("results"))

        while data.get("has_more"):
            query_dict["start_cursor"] = data.get("next_cursor")
            res = requests.post(
                DATABASE_URL_TMPL.format(database_id=database_id),
                headers=self.headers,
                json=query_dict,
            )
            res.raise_for_status()
            data = res.json()
            pages.extend(data.get("results"))

        return [page["id"] for page in pages]

    def search(self, query: str) -> List[str]:
        """Search Notion page given a text query."""
        done = False
        next_cursor: Optional[str] = None
        page_ids = []
        while not done:
            query_dict = {
                "query": query,
            }
            if next_cursor is not None:
                query_dict["start_cursor"] = next_cursor
            res = requests.post(SEARCH_URL, headers=self.headers, json=query_dict)
            data = res.json()
            for result in data["results"]:
                page_id = result["id"]
                page_ids.append(page_id)

            if data["next_cursor"] is None:
                done = True
                break
            else:
                next_cursor = data["next_cursor"]
        return page_ids

    def load_data(
        self, page_ids: List[str] = [], database_id: Optional[str] = None
    ) -> List[Document]:
        """Load data from the input directory.

        Args:
            page_ids (List[str]): List of page ids to load.
            database_id (str): Database_id from which to load page ids.

        Returns:
            List[Document]: List of documents.

        """
        if not page_ids and not database_id:
            raise ValueError("Must specify either `page_ids` or `database_id`.")
        docs = []
        if database_id is not None:
            # get all the pages in the database
            page_ids = self.query_database(database_id)
            for page_id in page_ids:
                page_text = self.read_page(page_id)
                docs.append(
                    Document(
                        text=page_text, id_=page_id, extra_info={"page_id": page_id}
                    )
                )
        else:
            for page_id in page_ids:
                page_text = self.read_page(page_id)
                docs.append(
                    Document(
                        text=page_text, id_=page_id, extra_info={"page_id": page_id}
                    )
                )

        return docs

class_name classmethod #

class_name() -> str

Get the name identifier of the class.

Source code in llama-index-integrations/readers/llama-index-readers-notion/llama_index/readers/notion/base.py 
50
51
52
53
@classmethod
def class_name(cls) -> str:
    """Get the name identifier of the class."""
    return "NotionPageReader"

read_page #

read_page(page_id: str) -> str

Read a page.

Source code in llama-index-integrations/readers/llama-index-readers-notion/llama_index/readers/notion/base.py 
101
102
103
def read_page(self, page_id: str) -> str:
    """Read a page."""
    return self._read_block(page_id)

query_database #

query_database(database_id: str, query_dict: Dict[str, Any] = {'page_size': 100}) -> List[str]

Get all the pages from a Notion database.

Source code in llama-index-integrations/readers/llama-index-readers-notion/llama_index/readers/notion/base.py 
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
def query_database(
    self, database_id: str, query_dict: Dict[str, Any] = {"page_size": 100}
) -> List[str]:
    """Get all the pages from a Notion database."""
    pages = []

    res = requests.post(
        DATABASE_URL_TMPL.format(database_id=database_id),
        headers=self.headers,
        json=query_dict,
    )
    res.raise_for_status()
    data = res.json()

    pages.extend(data.get("results"))

    while data.get("has_more"):
        query_dict["start_cursor"] = data.get("next_cursor")
        res = requests.post(
            DATABASE_URL_TMPL.format(database_id=database_id),
            headers=self.headers,
            json=query_dict,
        )
        res.raise_for_status()
        data = res.json()
        pages.extend(data.get("results"))

    return [page["id"] for page in pages]

search #

search(query: str) -> List[str]

Search Notion page given a text query.

Source code in llama-index-integrations/readers/llama-index-readers-notion/llama_index/readers/notion/base.py 
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
def search(self, query: str) -> List[str]:
    """Search Notion page given a text query."""
    done = False
    next_cursor: Optional[str] = None
    page_ids = []
    while not done:
        query_dict = {
            "query": query,
        }
        if next_cursor is not None:
            query_dict["start_cursor"] = next_cursor
        res = requests.post(SEARCH_URL, headers=self.headers, json=query_dict)
        data = res.json()
        for result in data["results"]:
            page_id = result["id"]
            page_ids.append(page_id)

        if data["next_cursor"] is None:
            done = True
            break
        else:
            next_cursor = data["next_cursor"]
    return page_ids

load_data #

load_data(page_ids: List[str] = [], database_id: Optional[str] = None) -> List[Document]

Load data from the input directory.

Parameters:

Name Type Description Default
page_ids List[str]

List of page ids to load.

 []
database_id str

Database_id from which to load page ids.

 None

Returns:

Type Description
List[Document]

List[Document]: List of documents.
Source code in llama-index-integrations/readers/llama-index-readers-notion/llama_index/readers/notion/base.py 
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
def load_data(
    self, page_ids: List[str] = [], database_id: Optional[str] = None
) -> List[Document]:
    """Load data from the input directory.

    Args:
        page_ids (List[str]): List of page ids to load.
        database_id (str): Database_id from which to load page ids.

    Returns:
        List[Document]: List of documents.

    """
    if not page_ids and not database_id:
        raise ValueError("Must specify either `page_ids` or `database_id`.")
    docs = []
    if database_id is not None:
        # get all the pages in the database
        page_ids = self.query_database(database_id)
        for page_id in page_ids:
            page_text = self.read_page(page_id)
            docs.append(
                Document(
                    text=page_text, id_=page_id, extra_info={"page_id": page_id}
                )
            )
    else:
        for page_id in page_ids:
            page_text = self.read_page(page_id)
            docs.append(
                Document(
                    text=page_text, id_=page_id, extra_info={"page_id": page_id}
                )
            )

    return docs