“Building a Job Search Engine with Elasticsearch and Golang: A Complete Guide with Aggregation Functionality”
Elasticsearch is a distributed search and analytics engine that allows you to search, analyze, and visualize large amounts of data in real-time. It is built on top of the Apache Lucene search engine library, and provides a powerful and flexible search and indexing platform for a variety of applications.
Here’s an example of an industry-based application that uses Elasticsearch and Golang. This example is a search engine for a job board website. It indexes job postings and allows users to search for jobs based on various criteria, such as job title, location, and keywords. Adding Aggregate Query to get the average salary and number of job postings for each location.
- Define the job posting schema
First, define the schema for the job postings that will be indexed in Elasticsearch. The schema should include the fields that will be used for searching and filtering the job postings:
type JobPosting struct {
ID int `json:"id"`
Title string `json:"title"`
Company string `json:"company"`
Location string `json:"location"`
Salary int `json:"salary"`
Keywords string `json:"keywords"`
CreatedAt int64 `json:"created_at"`
}- Set up Elasticsearch
Create a function that initializes the Elasticsearch client and creates the job posting index:
func setupElasticsearch() (*elasticsearch.Client, error) {
cfg := elasticsearch.Config{
Addresses: []string{"http://localhost:9200"},
}
es, err := elasticsearch.NewClient(cfg)
if err != nil {
return nil, err
}
// Create the index mapping
indexName := "job_postings"
mapping := `{
"mappings": {
"properties": {
"id": {
"type": "integer"
},
"title": {
"type": "text"
},
"company": {
"type": "text"
},
"location": {
"type": "text"
},
"salary": {
"type": "integer"
},
"keywords": {
"type": "text"
},
"created_at": {
"type": "date"
}
}
}
}`
// Create the index
req := esapi.IndicesCreateRequest{
Index: indexName,
Body: strings.NewReader(mapping),
}
res, err := req.Do(context.Background(), es)
if err != nil {
return nil, err
}
defer res.Body.Close()
if res.IsError() {
return nil, fmt.Errorf("failed to create index: %s", res.String())
}
return es, nil
}3. Index job postings
Create a function that adds job postings to the Elasticsearch index:
func indexJobPosting(es *elasticsearch.Client, indexName string, posting JobPosting) error {
req := esapi.IndexRequest{
Index: indexName,
DocumentID: strconv.Itoa(posting.ID),
Body: strings.NewReader(postingToJSON(posting)),
}
res, err := req.Do(context.Background(), es)
if err != nil {
return err
}
defer res.Body.Close()
if res.IsError() {
return fmt.Errorf("failed to index job posting: %s", res.String())
}
return nil
}
func postingToJSON(posting JobPosting) string {
jsonStr := `{
"id": %d,
"title": "%s",
"company": "%s",
"location": "%s",
"salary": %d,
"keywords": "%s",
"created_at": %d
}`
return fmt.Sprintf(jsonStr, posting.ID, posting.Title, posting.Company, posting.Location, posting.Salary, posting.Keywords, posting.CreatedAt)
}4. Search for job postings
Create a function that searches for job postings based on the user’s search query:
func searchJobPostings(es *elasticsearch.Client, indexName string, query string) ([]JobPosting, error) {
var results []JobPosting
req := esapi.SearchRequest{
Index: []string{indexName},
Body: buildSearchQuery(query),
}
res, err := req.Do(context.Background(), es)
if err != nil {
return nil, err
}
defer res.Body.Close()
if res.IsError() {
return nil, fmt.Errorf("search request failed: %s", res.String())
}
var response map[string]interface{}
if err := json.NewDecoder(res.Body).Decode(&response); err != nil {
return nil, err
}
// Extract aggregation results
aggregations := response["aggregations"].(map[string]interface{})
locationAggregation := aggregations["location"].(map[string]interface{})
locationBuckets := locationAggregation["buckets"].([]interface{})
hits := response["hits"].(map[string]interface{})["hits"].([]interface{})
for _, hit := range hits {
source := hit.(map[string]interface{})["_source"].(map[string]interface{})
posting := JobPosting{
ID: int(source["id"].(float64)),
Title: source["title"].(string),
Company: source["company"].(string),
Location: source["location"].(string),
Salary: int(source["salary"].(float64)),
Keywords: source["keywords"].(string),
CreatedAt: int64(source["created_at"].(float64)),
}
results = append(results, posting)
}
return results, parseAggregationResults(locationBuckets), nil
}
func parseAggregationResults(locationBuckets []interface{}) map[string]map[string]interface{} {
aggregationResults := make(map[string]map[string]interface{})
for _, bucket := range locationBuckets {
b := bucket.(map[string]interface{})
location := b["key"].(string)
aggregationResults[location] = map[string]interface{}{
"job_count": b["doc_count"].(float64),
"avg_salary": b["avg_salary"].(map[string]interface{})["value"].(float64)
}
return aggregationResults
}
func buildSearchQuery(query string) io.Reader {
searchQuery := map[string]interface{}{
"query": map[string]interface{}{
"multi_match": map[string]interface{}{
"query": query,
"fields": []string{"title^3", "keywords^2", "location", "company"},
},
},
}
body, _ := json.Marshal(searchQuery)
return bytes.NewReader(body)
}This function constructs a multi-match query that searches for the user’s query string in the job title, keywords, location, and company fields. It boosts the relevance of job postings that match the query string in the job title or keywords fields.
5. Define the Aggregation Query
The aggregation query that we’ll use in this example will be a composite aggregation with two sub-aggregations: a terms aggregation to group by location, and a metrics aggregation to calculate the average salary for each location.
func buildAggregationQuery() io.Reader {
aggregationQuery := map[string]interface{}{
"size": 0,
"aggs": map[string]interface{}{
"location": map[string]interface{}{
"terms": map[string]interface{}{
"field": "location.keyword",
},
"aggs": map[string]interface{}{
"avg_salary": map[string]interface{}{
"avg": map[string]interface{}{
"field": "salary",
},
},
},
},
},
}
body, _ := json.Marshal(aggregationQuery)
return bytes.NewReader(body)
}6. Putting it all together
Finally, the main function of the application puts everything together. It initializes the Elasticsearch client, indexes a few sample job postings, and then prompts the user to enter a search query:
func main() {
es, err := setupElasticsearch()
if err != nil {
log.Fatalf("failed to set up Elasticsearch: %v", err)
}
// Index some sample job postings
indexJobPosting(es, "job_postings", JobPosting{
ID: 1,
Title: "Software Engineer",
Company: "Acme Corp",
Location: "San Francisco, CA",
Salary: 100000,
Keywords: "Go, Elasticsearch",
CreatedAt: time.Now().Unix(),
})
indexJobPosting(es, "job_postings", JobPosting{
ID: 2,
Title: "Data Scientist",
Company: "Data Co",
Location: "New York, NY",
Salary: 120000,
Keywords: "Python, Machine Learning, Elasticsearch",
CreatedAt: time.Now().Unix(),
})
// Prompt the user for a search query
reader := bufio.NewReader(os.Stdin)
fmt.Print("Enter a search query: ")
query, _ := reader.ReadString('\n')
// Search for job postings
results, aggregationResults, err := searchJobPostings(es, "job_postings", query)
if err != nil {
log.Fatalf("search failed: %v ",err)
}
// Print the search results
fmt.Printf("Search results for '%s':\n", query)
for _, result := range results {
fmt.Printf("ID: %d\nTitle: %s\nCompany: %s\nLocation: %s\nSalary: %d\nKeywords: %s\nCreated At: %s\n\n",
result.ID, result.Title, result.Company, result.Location, result.Salary, result.Keywords, time.Unix(result.CreatedAt, 0))
}
// Print aggregation results
fmt.Println("Aggregation results:")
for location, results := range aggregationResults {
fmt.Printf("Location: %s\nJob Count: %.0f\nAverage Salary: %.2f\n\n", location, results["job_count"].(float64), results["avg_salary"].(float64))
}
}This code initializes the Elasticsearch client using the `setupElasticsearch` function, which we defined earlier. It then indexes two sample job postings using the `indexJobPosting` function. Next, it prompts the user for a search query using the `bufio` package. It then calls the `searchJobPostings` function to execute the search query and returns the results.
Finally, the code prints the search results to the console. This application demonstrates how to use Elasticsearch with Go to build a simple job search engine. It covers many of the core concepts of Elasticsearch, including index creation, document indexing, and search queries.
