How to build a search system with nodejs and OpenSearch?

Once your product starts growing, the data accumulation increases. Searching is one of the important features that would be required for several reasons. Some of it may be searching for a product through a large product catalog, or profiles within a large dataset of profiles. You need to have a full-text search that can give results not in seconds but in milliseconds or less.

In this article, I explain how you can implement search functionality to your product using nodejs and AWS OpenSearch.

I have used the version 2 of AWS SDK. However, if you are using other versions this may act as a reference doc.

The code block doesn’t highlight syntax, the alignment may not be correct, and it’s due to the substack code block. You can copy and paste it to VSCode for better syntax highlight

const env = process.env.NODE_ENV || 'development';

const AWS = require('aws-sdk');

const { Client } = require('@opensearch-project/opensearch');
// eslint-disable-next-line import/no-unresolved
const { AwsSigv4Signer } = require('@opensearch-project/opensearch/aws');
const config = require('config.json')[env];

const OS_paramas = {
    secretAccessKey: config.AWS_OpenSearch.OS_SECRET,
    accessKeyId: config.AWS_OpenSearch.OS_KEY
};

AWS.config.update(OS_paramas);

const awsCredentials = () => new Promise((resolve, reject) => {
    AWS.config.getCredentials((err, credentials) => {
        if (err) {
            reject(err);
        } else {
            resolve(credentials);
        }
    });
});

const openSearchDomainUrl = DOMAIN_URL;

Now we will write a service to create, delete an index, add, and delete a document, perform a search, and execute raw SQL query.

Create Index

    createIndex: async (index_name) => {
        try {
            const settings = {
                settings: {
                    index: {},
                    // add mapings to your index
                    mappings: {
                        properties: {}
                    }

                },
            };
            const response = await client.indices.create({
                index: index_name,
                body: settings
            });
            return { success: true, message: response };
        } catch (error) {
            console.log('Error occurred while creating index', error);
            return { success: false, message: 'Error' };
        }
    }

Delete Index

    deleteIndex: async (index_name) => {
        try {
            await client.indices.delete({
                index: index_name
            });
            return { success: true, message: 'Index deleted' };
        } catch (error) {
            console.log('Error while deleting the index name', error);
            return { success: false, message: 'Error' };
        }
    }

Add document

    addDocument: async (document, index_name, type) => {
        try {
            if (type === 'single') {
                await client.index({
                    index: index_name,
                    body: document,
                    refresh: true
                });
                return { success: true, message: 'added to index' };
            }

            if (type === 'bulk') {
                await client.helpers.bulk({
                    datasource: document,
                    onDocument() {
                        return {
                            index: { _index: index_name }
                        };
                    }
                });
                return { success: true, message: 'added to index' };
            }

            return { success: false, message: 'Type is missing' };
        } catch (error) {
            console.log('Error while adding the document', error);
            return { success: false, message: 'Error' };
        }
    }

Delete Document

/**
 * @description delete a document within index,
 * Do not pass the document id, pass corresponding _id that you have generated manually or automatically generated by opensearch
 * @param {string} index_name
 * @param {string} _id this is generated by the opensearch (else you need to generate manually while adding
 * Reference: https://docs.aws.amazon.com/opensearch-service/latest/developerguide/quick-start.html)
 * @returns {object} response object
*/

    deleteDocument: async (index_name, _id) => {
        try {
            await client.delete({
                index: index_name,
                id: _id
            });
            return { success: true, message: 'Successfully deleted' };
        } catch (error) {
            console.log('Error while deleting the document', error);
            return { success: false, message: 'Error while deleting' };
        }
    }

Update Document (Bulk)

 /**
  * @description updateDocumentInBulk can be used to update multiple or individual documents
  * @param {array} document data must be in array of objects
  * @param {string} index_name
  * @param {string} _id
  * @returns {object}
 */
    updateDocumentInBulk: async (document, index_name, _id) => {
        try {
            await client.helpers.bulk({
                datasource: document,
                onDocument() {
                    return [
                        {
                            update: { _index: index_name, _id }
                        },
                        {
                            doc_as_upsert: true
                        }
                    ];
                }
            });

            return { success: true, message: 'Document updated' };
        } catch (error) {
            console.log('Error while updating the document', error);
            return { success: false, message: 'Error while updating' };
        }
    },

Search

     search: async () => {
        try {
     
            const query = {
                query: {
                    match: {
                        YOUR_COLUMN_NAME: {
                            query: ''
                        }
                    },
                },
            };

            const response = await client.search({
                index: INDEX_NAME,
                body: query
            });
            return { success: true, message: response };
        } catch (error) {
            console.log('Error while performing the search', error);
            return { success: false, message: 'Error while searching' };
        }
    }

Query

I will be using curl, but you can use REST API (Requires proper authorization before making any request)

    query: async (query) => {
        try {
            const util = require('node:util');
            const exec = util.promisify(require('node:child_process').exec);
            const queryObj = {
                query
            };
            const username = config.AWS_OpenSearch.username;
            const password = config.AWS_OpenSearch.password;
            
            const queryObjStringified = JSON.stringify(queryObj);
            
            const execCurl = `curl -XPOST DOMAIN_URL/_plugins/_sql -u ${username}:${password} -k -H 'Content-Type: application/json' -d '${queryObjStringified}'`;
            
           const { stdout } = await exec(execCurl);
           const parsedOutput = JSON.parse(stdout);
           return { success: true, message: parsedOutput };
        } catch (error) {
            console.log('Error while performing query on opensearch', error);
            return { success: false, message: 'Error while performing the query' };
        }
    }