Compare commits

...

4 Commits

Author SHA1 Message Date
HarryPaulo
9a3c8a3c11 fix: new url for frankfurter.app, now use api.frankfurter.app
Signed-off-by: Akhil Narang <me@akhilnarang.dev>
2024-10-02 11:21:40 +05:30
Akhil Narang
13f5634570 chore: resolve conflicts
Signed-off-by: Akhil Narang <me@akhilnarang.dev>
2024-10-01 17:29:39 +05:30
Ankush Menat
7ff97794f4 fix: encode thumbnail URL
If it contains space the URL won't load

(cherry picked from commit 30039e8e62)
2024-10-01 11:50:42 +00:00
Ankush Menat
9ac973fff6 refactor!: drop redisearch
incr: replace text and tag fields

incr: use rediswrapper's make key

incr: indexDefinition from redis

incr: replace index creation

incr: replace AutoCompleter

incr: replace product search ac

incr: replace client querying

fix: broken redisearch load test

fix: pass actual query to get suggestion
(cherry picked from commit 4a38ce659d)

# Conflicts:
#	erpnext/e_commerce/redisearch_utils.py
#	erpnext/templates/pages/product_search.py
#	pyproject.toml
2024-10-01 11:50:42 +00:00
5 changed files with 39 additions and 41 deletions

View File

@@ -84,7 +84,7 @@ def get_api_endpoint(service_provider: str | None = None, use_http: bool = False
if service_provider == "exchangerate.host":
api = "api.exchangerate.host/convert"
elif service_provider == "frankfurter.app":
api = "frankfurter.app/{transaction_date}"
api = "api.frankfurter.app/{transaction_date}"
protocol = "https://"
if use_http:

View File

@@ -221,7 +221,7 @@ erpnext.ProductSearch = class {
let thumbnail = res.thumbnail || "/assets/erpnext/images/ui-states/cart-empty-state.png";
html += `
<div class="dropdown-item" style="display: flex;">
<img class="item-thumb col-2" src=${thumbnail} />
<img class="item-thumb col-2" src=${encodeURI(thumbnail)} />
<div class="col-9" style="white-space: normal;">
<a href="/${res.route}">${res.web_item_name}</a><br>
<span class="brand-line">${res.brand ? "by " + res.brand : ""}</span>

View File

@@ -7,7 +7,9 @@ import frappe
from frappe import _
from frappe.utils.redis_wrapper import RedisWrapper
from redis import ResponseError
from redisearch import AutoCompleter, Client, IndexDefinition, Suggestion, TagField, TextField
from redis.commands.search.field import TagField, TextField
from redis.commands.search.indexDefinition import IndexDefinition
from redis.commands.search.suggestion import Suggestion
WEBSITE_ITEM_INDEX = "website_items_index"
WEBSITE_ITEM_KEY_PREFIX = "website_item:"
@@ -35,12 +37,9 @@ def is_redisearch_enabled():
def is_search_module_loaded():
try:
cache = frappe.cache()
out = cache.execute_command("MODULE LIST")
parsed_output = " ".join(
" ".join([frappe.as_unicode(s) for s in o if not isinstance(s, int)]) for o in out
)
return "search" in parsed_output
for module in cache.module_list():
if module.get(b"name") == b"search":
return True
except Exception:
return False # handling older redis versions
@@ -58,18 +57,18 @@ def if_redisearch_enabled(function):
def make_key(key):
return f"{frappe.conf.db_name}|{key}".encode()
return frappe.cache().make_key(key)
@if_redisearch_enabled
def create_website_items_index():
"Creates Index Definition."
# CREATE index
client = Client(make_key(WEBSITE_ITEM_INDEX), conn=frappe.cache())
redis = frappe.cache()
index = redis.ft(WEBSITE_ITEM_INDEX)
try:
client.drop_index() # drop if already exists
index.dropindex() # drop if already exists
except ResponseError:
# will most likely raise a ResponseError if index does not exist
# ignore and create index
@@ -86,9 +85,9 @@ def create_website_items_index():
if "web_item_name" in idx_fields:
idx_fields.remove("web_item_name")
idx_fields = list(map(to_search_field, idx_fields))
idx_fields = [to_search_field(f) for f in idx_fields]
client.create_index(
index.create_index(
[TextField("web_item_name", sortable=True), *idx_fields],
definition=idx_def,
)
@@ -119,8 +118,8 @@ def insert_item_to_index(website_item_doc):
@if_redisearch_enabled
def insert_to_name_ac(web_name, doc_name):
ac = AutoCompleter(make_key(WEBSITE_ITEM_NAME_AUTOCOMPLETE), conn=frappe.cache())
ac.add_suggestions(Suggestion(web_name, payload=doc_name))
ac = frappe.cache().ft()
ac.sugadd(WEBSITE_ITEM_NAME_AUTOCOMPLETE, Suggestion(web_name, payload=doc_name))
def create_web_item_map(website_item_doc):
@@ -157,9 +156,8 @@ def delete_item_from_index(website_item_doc):
@if_redisearch_enabled
def delete_from_ac_dict(website_item_doc):
"""Removes this items's name from autocomplete dictionary"""
cache = frappe.cache()
name_ac = AutoCompleter(make_key(WEBSITE_ITEM_NAME_AUTOCOMPLETE), conn=cache)
name_ac.delete(website_item_doc.web_item_name)
ac = frappe.cache().ft()
ac.sugdel(website_item_doc.web_item_name)
@if_redisearch_enabled
@@ -170,8 +168,6 @@ def define_autocomplete_dictionary():
"""
cache = frappe.cache()
item_ac = AutoCompleter(make_key(WEBSITE_ITEM_NAME_AUTOCOMPLETE), conn=cache)
item_group_ac = AutoCompleter(make_key(WEBSITE_ITEM_CATEGORY_AUTOCOMPLETE), conn=cache)
# Delete both autocomplete dicts
try:
@@ -180,36 +176,40 @@ def define_autocomplete_dictionary():
except Exception:
raise_redisearch_error()
create_items_autocomplete_dict(autocompleter=item_ac)
create_item_groups_autocomplete_dict(autocompleter=item_group_ac)
create_items_autocomplete_dict()
create_item_groups_autocomplete_dict()
@if_redisearch_enabled
def create_items_autocomplete_dict(autocompleter):
def create_items_autocomplete_dict():
"Add items as suggestions in Autocompleter."
ac = frappe.cache().ft()
items = frappe.get_all("Website Item", fields=["web_item_name", "item_group"], filters={"published": 1})
for item in items:
autocompleter.add_suggestions(Suggestion(item.web_item_name))
ac.sugadd(WEBSITE_ITEM_NAME_AUTOCOMPLETE, Suggestion(item.web_item_name))
@if_redisearch_enabled
def create_item_groups_autocomplete_dict(autocompleter):
def create_item_groups_autocomplete_dict():
"Add item groups with weightage as suggestions in Autocompleter."
published_item_groups = frappe.get_all(
"Item Group", fields=["name", "route", "weightage"], filters={"show_in_website": 1}
)
if not published_item_groups:
return
ac = frappe.cache().ft()
for item_group in published_item_groups:
payload = json.dumps({"name": item_group.name, "route": item_group.route})
autocompleter.add_suggestions(
ac.sugadd(
WEBSITE_ITEM_CATEGORY_AUTOCOMPLETE,
Suggestion(
string=item_group.name,
score=frappe.utils.flt(item_group.weightage) or 1.0,
payload=payload, # additional info that can be retrieved later
)
),
)

View File

@@ -5,14 +5,13 @@ import json
import frappe
from frappe.utils import cint, cstr
from redisearch import AutoCompleter, Client, Query
from redis.commands.search.query import Query
from erpnext.e_commerce.redisearch_utils import (
WEBSITE_ITEM_CATEGORY_AUTOCOMPLETE,
WEBSITE_ITEM_INDEX,
WEBSITE_ITEM_NAME_AUTOCOMPLETE,
is_redisearch_enabled,
make_key,
)
from erpnext.e_commerce.shopping_cart.product_info import set_product_info_for_website
from erpnext.setup.doctype.item_group.item_group import get_item_for_list_in_html
@@ -85,17 +84,17 @@ def product_search(query, limit=10, fuzzy_search=True):
if not query:
return search_results
red = frappe.cache()
redis = frappe.cache()
query = clean_up_query(query)
# TODO: Check perf/correctness with Suggestions & Query vs only Query
# TODO: Use Levenshtein Distance in Query (max=3)
ac = AutoCompleter(make_key(WEBSITE_ITEM_NAME_AUTOCOMPLETE), conn=red)
client = Client(make_key(WEBSITE_ITEM_INDEX), conn=red)
suggestions = ac.get_suggestions(
redisearch = redis.ft(WEBSITE_ITEM_INDEX)
suggestions = redisearch.sugget(
WEBSITE_ITEM_NAME_AUTOCOMPLETE,
query,
num=limit,
fuzzy=fuzzy_search and len(query) > 3, # Fuzzy on length < 3 can be real slow
fuzzy=fuzzy_search and len(query) > 3,
)
# Build a query
@@ -105,8 +104,8 @@ def product_search(query, limit=10, fuzzy_search=True):
query_string += f"|('{clean_up_query(s.string)}')"
q = Query(query_string)
results = redisearch.search(q)
results = client.search(q)
search_results["results"] = list(map(convert_to_dict, results.docs))
search_results["results"] = sorted(
search_results["results"], key=lambda k: frappe.utils.cint(k["ranking"]), reverse=True
@@ -140,8 +139,8 @@ def get_category_suggestions(query):
if not query:
return search_results
ac = AutoCompleter(make_key(WEBSITE_ITEM_CATEGORY_AUTOCOMPLETE), conn=frappe.cache())
suggestions = ac.get_suggestions(query, num=10, with_payloads=True)
ac = frappe.cache().ft()
suggestions = ac.sugget(WEBSITE_ITEM_CATEGORY_AUTOCOMPLETE, query, num=10, with_payloads=True)
results = [json.loads(s.payload) for s in suggestions]

View File

@@ -12,7 +12,6 @@ dependencies = [
"pycountry~=20.7.3",
"python-stdnum~=1.16",
"Unidecode~=1.2.0",
"redisearch~=2.1.0",
"rapidfuzz~=2.15.0",
"holidays~=0.28",