# -*- coding: utf-8 -*-
from common import *
import random
def test_1282(env):
conn = getConnectionByEnv(env)
env.expect('FT.CREATE', 'idx', 'ON', 'HASH', 'SCHEMA', 'txt1', 'TEXT').ok()
env.assertEqual(conn.execute_command('hset', 'doc1', 'txt1', 'foo'), 1)
# optional search for new word would crash server
env.expect('FT.SEARCH idx', '~foo').equal([1, 'doc1', ['txt1', 'foo']])
env.expect('FT.SEARCH idx', '~bar ~foo').equal([1, 'doc1', ['txt1', 'foo']])
def test_1304(env):
env.expect('FT.CREATE idx SCHEMA txt1 TEXT').equal('OK')
env.expect('FT.EXPLAIN idx -20*').equal('PREFIX{-20*}\n')
env.expect('FT.EXPLAIN idx -\\20*').equal('NOT{\n PREFIX{20*}\n}\n')
@skip(cluster=True)
def test_10140_phonetic_after_non_text_field(env):
env.expect('FT.CREATE', 'idx', 'ON', 'HASH', 'PREFIX', '1', 'phonetic:',
'SCHEMA', 'filingDate', 'TAG', 'chunkText', 'TEXT', 'PHONETIC', 'dm:en').ok()
env.cmd('HSET', 'phonetic:1', 'chunkText', 'cash', 'filingDate', '2026-06-15')
env.expect('FT.SEARCH', 'idx', '@chunkText:kash', 'NOCONTENT').equal([1, 'phonetic:1'])
@skip(cluster=True)
def test_10140_phonetic_after_noindex_text_field(env):
env.expect('FT.CREATE', 'idx', 'ON', 'HASH', 'PREFIX', '1', 'phonetic:',
'SCHEMA', 'ignored', 'TEXT', 'NOINDEX',
'chunkText', 'TEXT', 'PHONETIC', 'dm:en').ok()
env.cmd('HSET', 'phonetic:1', 'ignored', 'skip me', 'chunkText', 'cash')
env.expect('FT.SEARCH', 'idx', '@chunkText:kash', 'NOCONTENT').equal([1, 'phonetic:1'])
env.expect('FT.SEARCH', 'idx', '@chunkText:(kash)=>{$phonetic:true}', 'NOCONTENT') \
.equal([1, 'phonetic:1'])
@skip(cluster=True)
def test_10140_empty_query_after_noindex_text_field(env):
env.expect('FT.CREATE', 'idx', 'SCHEMA', 'ignored', 'TEXT', 'NOINDEX', 'text', 'TEXT').ok()
env.expect('FT.SEARCH', 'idx', '@text:("")') \
.error().contains('Use `INDEXEMPTY` in field creation')
@skip(cluster=True, no_json=True)
def test_10140_slop_after_noindex_text_field(env):
env.expect('FT.CREATE', 'idx', 'ON', 'JSON', 'SCHEMA',
'$.ignored', 'AS', 'ignored', 'TEXT', 'NOINDEX',
'$.text[*]', 'AS', 'text', 'TEXT').ok()
env.expect('FT.SEARCH', 'idx', '@text:(hello world)=>{$slop:1}') \
.error().contains('with undefined ordering')
@skip(cluster=True)
def test_1414(env):
env.expect('FT.CREATE idx SCHEMA txt1 TEXT').equal('OK')
env.cmd('hset', 'doc', 'foo', 'hello', 'bar', 'world')
env.expect('ft.search', 'idx', '*', 'limit', '0', '1234567').error().contains('LIMIT exceeds maximum of 1000000')
env.expect(config_cmd(), 'set', 'MAXSEARCHRESULTS', '-1').ok()
env.assertEqual(toSortedFlatList(env.cmd('ft.search', 'idx', '*', 'limit', '0', '1234567')),
toSortedFlatList([1, 'doc', ['foo', 'hello', 'bar', 'world']]))
env.expect(config_cmd(), 'set', 'MAXSEARCHRESULTS', '1000000').ok()
def test_1502(env):
conn = getConnectionByEnv(env)
conn.execute_command('HSET', 'a', 'bar', 'hello')
env.expect('FT.CREATE idx1 SKIPINITIALSCAN SCHEMA foo TEXT').ok()
env.expect('FT.CREATE idx2 SKIPINITIALSCAN SCHEMA foo TEXT').ok()
env.expect('ft.search idx1 *').equal([0])
env.expect('ft.search idx2 *').equal([0])
env.expect('FT.ALTER idx1 SKIPINITIALSCAN SCHEMA ADD bar TEXT').ok()
env.expect('FT.ALTER idx2 SCHEMA ADD bar TEXT').ok()
waitForIndex(env, 'idx2')
env.expect('ft.search idx1 *').equal([0])
env.expect('ft.search idx2 *').equal([1, 'a', ['bar', 'hello']])
def test_1601(env):
conn = getConnectionByEnv(env)
env.cmd('FT.CREATE', 'idx:movie', 'SCHEMA', 'title', 'TEXT')
conn.execute_command('HSET', 'movie:1', 'title', 'Star Wars: Episode I - The Phantom Menace')
conn.execute_command('HSET', 'movie:2', 'title', 'Star Wars: Episodes II - Attack of the Clones')
conn.execute_command('HSET', 'movie:3', 'title', 'Star Wars: Episode III - Revenge of the Sith')
res = env.cmd('ft.search idx:movie @title:(episode) withscores nocontent')
env.assertEqual(res[0], 3)
def testMultiSortby(env):
conn = getConnectionByEnv(env)
env.cmd('FT.CREATE', 'idx', 'SCHEMA', 't1', 'TEXT', 'SORTABLE', 't2', 'TEXT', 'SORTABLE', 't3', 'TEXT', 'SORTABLE')
conn.execute_command('hset', '1', 't1', 'foo', 't2', 'bar', 't3', 'baz')
conn.execute_command('hset', '2', 't1', 'bar', 't2', 'foo', 't3', 'baz')
sortby_t1 = [2, '2', '1']
sortby_t2 = [2, '1', '2']
env.expect('ft.search idx foo nocontent sortby t1 asc').equal(sortby_t1)
env.expect('ft.search idx foo nocontent sortby t2 asc').equal(sortby_t2)
env.expect('ft.search idx foo nocontent sortby t1 sortby t3').error()\
.contains('Multiple SORTBY steps are not allowed')
env.expect('ft.aggregate idx foo nocontent sortby 2 @t1 asc sortby 2 @t3 desc').error()\
.contains('Multiple SORTBY steps are not allowed. Sort multiple fields in a single step')
#TODO: allow multiple sortby steps
#env.expect('ft.search idx foo nocontent sortby t1 sortby t3').equal(sortby_t1)
#env.expect('ft.search idx foo nocontent sortby t2 sortby t3').equal(sortby_t2)
def test_1667(env):
conn = getConnectionByEnv(env)
env.cmd('FT.CREATE', 'idx', 'SCHEMA', 'tag', 'TAG', 'text', 'TEXT')
env.expect('ft.search idx @tag:{a}').equal([0])
env.expect('ft.search idx @tag:{b}').equal([0])
conn.execute_command('HSET', 'doc', 'tag', 'a,b')
conn.execute_command('HSET', 'doc1', 'tag', 'abc')
# test single stopword
env.expect('ft.search idx @tag:{a}').equal([1, 'doc', ['tag', 'a,b']])
env.expect('ft.search idx @tag:{b}').equal([1, 'doc', ['tag', 'a,b']])
env.expect('ft.search idx @tag:{c}').equal([0])
# test stopword in list
env.expect('ft.search idx @tag:{a|c}').equal([1, 'doc', ['tag', 'a,b']])
env.expect('ft.search idx @tag:{c|a}').equal([1, 'doc', ['tag', 'a,b']])
env.expect('ft.search idx @tag:{c|a|c}').equal([1, 'doc', ['tag', 'a,b']])
# test stopword with prefix
env.expect('ft.search idx @tag:{ab*}').equal([1, 'doc1', ['tag', 'abc']])
env.expect('ft.search idx @tag:{abc*}').equal([1, 'doc1', ['tag', 'abc']])
# ensure regular text field
conn.execute_command('HSET', 'doc_a', 'text', 'a')
conn.execute_command('HSET', 'doc_b', 'text', 'b')
env.expect('ft.search idx a').equal([0])
env.expect('ft.search idx b').equal([1, 'doc_b', ['text', 'b']])
@skip(cluster=False)
def test_MOD_7454(env: Env):
env.cmd('FT.CREATE', 'idx', 'SCHEMA', 'n', 'NUMERIC')
n_docs = 1100 * env.shardsCount # We need more than 1000 docs in each shard
with env.getClusterConnectionIfNeeded() as conn:
for i in range(n_docs):
conn.execute_command('HSET', f'doc{i}', 'n', i)
# We have more than 1000 docs in each shard, and the query should return all of them.
# In cluster mode, FT.AGGREGATE (and FT.PROFILE AGGREGATE) uses cursors in each shard, reading
# 1000 at a time and then aggregate the replies.
# The second batch read from each shard will require to "reopen" the numeric index (on each shard).
# We have a large numeric range so we expect to have a union iterator of multiple numeric iterators.
# We are also in PROFILE mode, so each numeric iterator should be wrapped with a PROFILE iterator.
# The issue was that we casted each iterator in the union to a numeric iterator without checking if it's
# a PROFILE iterator, which caused a crash.
# We first validate that the query returns without error.
res = env.expect('FT.PROFILE', 'idx', 'AGGREGATE', 'QUERY', f'@n:[0 {n_docs}]').noError().res
# With the given setup we should have enough docs to trigger the issue.
# Let's validate that we got a union iterator of multiple numeric iterators.
union_profile = to_dict(to_dict(res[-1][1][0])['Iterators profile']) # take the first shard info
env.assertEqual(union_profile['Type'], 'UNION')
env.assertEqual(union_profile['Query type'], 'NUMERIC')
def test_MOD_865(env):
conn = getConnectionByEnv(env)
args_list = ['FT.CREATE', 'idx', 'SCHEMA']
for i in range(1025):
args_list.extend([i, 'NUMERIC', 'SORTABLE'])
env.expect(*args_list).error().contains('Schema is limited to 1024 fields')
env.expect('FT.DROPINDEX', 'idx')
args_list = ['FT.CREATE', 'idx', 'SCHEMA']
for i in range(129):
args_list.extend([i, 'TEXT'])
env.expect(*args_list).error().contains(f'Schema is limited to {arch_int_bits()} TEXT fields')
env.expect('FT.DROPINDEX', 'idx')
args_list = ['FT.CREATE', 'idx', 'SCHEMA']
for i in range(2):
args_list.extend(['txt', 'TEXT'])
env.expect(*args_list).error().contains('Duplicate field in schema - txt')
env.expect('FT.DROPINDEX', 'idx')
def test_MOD_6411(env):
# FT.CREATE used to crash on a stack overflow when the argument list was
# large enough (the parser allocated a VLA of `const char *` on the stack).
# The parser now consumes RedisModuleString ** directly through ArgsCursor,
# so an oversized field list is rejected with the standard schema-limit
# error instead of crashing the server.
args_list = ['FT.CREATE', 'idx', 'SCHEMA']
for i in range(100000):
args_list.extend([f'field{i}', 'NUMERIC', 'SORTABLE'])
env.expect(*args_list).error().contains('Schema is limited to 1024 fields')
env.expect('FT.DROPINDEX', 'idx')
def test_issue1826(env):
# Stopword query is case sensitive.
conn = getConnectionByEnv(env)
env.cmd('FT.CREATE', 'idx', 'SCHEMA', 't', 'TEXT')
conn.execute_command('HSET', 'doc', 't', 'boy with glasses')
env.expect('FT.SEARCH', 'idx', 'boy with glasses').equal([1, 'doc', ['t', 'boy with glasses']])
env.expect('FT.SEARCH', 'idx', 'boy With glasses').equal([1, 'doc', ['t', 'boy with glasses']])
def test_issue1834(env):
# Stopword query is case sensitive.
conn = getConnectionByEnv(env)
env.cmd('FT.CREATE', 'idx', 'SCHEMA', 't', 'TEXT')
conn.execute_command('HSET', 'doc', 't', 'hell hello')
env.expect('FT.SEARCH', 'idx', 'hell|hello', 'HIGHLIGHT').equal([1, 'doc', ['t', 'hell hello']])
@skip(cluster=True)
def test_issue1880(env):
# order of iterator in intersect is optimized by function
conn = getConnectionByEnv(env)
env.cmd(config_cmd(), 'SET', '_PRINT_PROFILE_CLOCK', 'false')
env.cmd('FT.CREATE', 'idx', 'SCHEMA', 't', 'TEXT')
conn.execute_command('HSET', 'doc1', 't', 'hello world')
conn.execute_command('HSET', 'doc2', 't', 'hello')
excepted_res = ['Type', 'INTERSECT', 'Number of reading operations', 1, 'Child iterators', [
['Type', 'TEXT', 'Term', 'world', 'Number of reading operations', 1, 'Estimated number of matches', 1],
['Type', 'TEXT', 'Term', 'hello', 'Number of reading operations', 1, 'Estimated number of matches', 2]]]
res1 = env.cmd('FT.PROFILE', 'idx', 'SEARCH', 'QUERY', 'hello world')
res2 = env.cmd('FT.PROFILE', 'idx', 'SEARCH', 'QUERY', 'world hello')
# both queries return `world` iterator before `hello`
env.assertEqual(res1[1][1][0][3], excepted_res)
env.assertEqual(res2[1][1][0][3], excepted_res)
# test with a term which does not exist
excepted_res = ['Type', 'EMPTY', 'Number of reading operations', 0]
res3 = env.cmd('FT.PROFILE', 'idx', 'SEARCH', 'QUERY', 'hello new world')
env.assertEqual(res3[1][1][0][3], excepted_res)
def test_issue1932(env):
conn = getConnectionByEnv(env)
env.cmd('FT.CREATE', 'idx', 'SCHEMA', 't', 'TEXT')
env.expect('FT.AGGREGATE', 'idx', '*', 'LIMIT', '100000000000000000', '1000000', 'SORTBY', '1', '@t').error() \
.contains('OFFSET exceeds maximum of 1000000')
env.expect('FT.AGGREGATE', 'idx', '*', 'LIMIT', '1000000', '100000000000000000', 'SORTBY', '1', '@t').error() \
.contains('LIMIT exceeds maximum of 2147483648')
@skip(cluster=True)
def test_MOD_14655(env:Env):
env.expect('FT.CREATE', 'idx', 'NOFIELDS', 'MAXTEXTFIELDS', 'SCHEMA', 't', 'TEXT').error() \
.contains('MAXTEXTFIELDS cannot be used with NOFIELDS')
# Previously, if the index was created successfully, the following HSET will cause a crash.
with env.getClusterConnectionIfNeeded() as conn:
conn.execute_command('HSET', 'doc1', 't', 'hello world')
def test_issue1988(env):
conn = getConnectionByEnv(env)
env.cmd('FT.CREATE', 'idx', 'SCHEMA', 't', 'TEXT')
conn.execute_command('HSET', 'doc1', 't', 'foo')
score = '0.28768207245178085'
env.expect('FT.SEARCH', 'idx', 'foo').equal([1, 'doc1', ['t', 'foo']])
env.expect('FT.SEARCH', 'idx', 'foo', 'WITHSCORES').equal([1, 'doc1', score, ['t', 'foo']])
env.expect('FT.SEARCH', 'idx', 'foo', 'SORTBY' , 't').equal([1, 'doc1', ['t', 'foo']])
env.expect('FT.SEARCH', 'idx', 'foo', 'WITHSCORES', 'SORTBY' , 't').equal([1, 'doc1', score, ['t', 'foo']])
@no_msan
def testIssue2104Hash(env):
# 'AS' attribute does not work in functions
conn = getConnectionByEnv(env)
# hash
env.cmd('FT.CREATE', 'hash_idx', 'SCHEMA', 'name', 'TEXT', 'SORTABLE', 'subj1', 'NUMERIC', 'SORTABLE')
conn.execute_command('hset', 'data1','name', 'abc', 'subj1', '20')
# load a single field
env.expect('FT.AGGREGATE', 'hash_idx', '*', 'LOAD', '1', '@subj1') \
.equal([1, ['subj1', '20']])
# load a field with an attribute
env.expect('FT.AGGREGATE', 'hash_idx', '*', 'LOAD', '3', '@subj1', 'AS', 'a') \
.equal([1, ['a', '20']])
# load field and use `APPLY`
env.expect('FT.AGGREGATE', 'hash_idx', '*', 'LOAD', '3', '@subj1', 'AS', 'a', 'APPLY', '(@a+@a)/2', 'AS', 'avg') \
.equal([1, ['a', '20', 'avg', '20']])
# load a field implicitly with `APPLY`
res = env.cmd('FT.AGGREGATE', 'hash_idx', '*', 'APPLY', '(@subj1+@subj1)/2', 'AS', 'avg')
env.assertEqual(toSortedFlatList([1, ['subj1', '20', 'avg', '20']]), toSortedFlatList(res))
res = env.cmd('FT.AGGREGATE', 'hash_idx', '*', 'LOAD', '3', '@subj1', 'AS', 'a', 'APPLY', '(@subj1+@subj1)/2', 'AS', 'avg')
env.assertEqual(toSortedFlatList([1, ['a', '20', 'subj1', '20', 'avg', '20']]), toSortedFlatList(res))
@skip(msan=True, no_json=True)
def testIssue2104JSON(env):
# 'AS' attribute does not work in functions
conn = getConnectionByEnv(env)
env.cmd('FT.CREATE', 'json_idx', 'ON', 'JSON', 'SCHEMA', '$.name', 'AS', 'name', 'TEXT', 'SORTABLE',
'$.subj1', 'AS', 'subj2', 'NUMERIC', 'SORTABLE')
env.cmd('JSON.SET', 'doc:1', '$', r'{"name":"Redis", "subj1":3.14}')
env.expect('json.get', 'doc:1', '$').equal('[{"name":"Redis","subj1":3.14}]')
# load a single field
env.expect('FT.AGGREGATE', 'json_idx', '*', 'LOAD', '1', '@subj2') \
.equal([1, ['subj2', '3.14']])
# load a field with an attribute
env.expect('FT.AGGREGATE', 'json_idx', '*', 'LOAD', '3', '@subj2', 'AS', 'a') \
.equal([1, ['a', '3.14']])
# load field and use `APPLY`
env.expect('FT.AGGREGATE', 'json_idx', '*', 'LOAD', '3', '@subj2', 'AS', 'a', 'APPLY', '(@a+@a)/2', 'AS', 'avg') \
.equal([1, ['a', '3.14', 'avg', '3.14']])
# load a field implicitly with `APPLY`
res = env.cmd('FT.AGGREGATE', 'json_idx', '*', 'APPLY', '(@subj2+@subj2)/2', 'AS', 'avg')
env.assertEqual(toSortedFlatList([1, ['subj2', '3.14', 'avg', '3.14']]), toSortedFlatList(res))
# load a field with an attribute
env.expect('FT.AGGREGATE', 'json_idx', '*', 'LOAD', '3', '@$.subj1', 'AS', 'a') \
.equal([1, ['a', '3.14']])
# In this example we get both `a` and `subj1` since
env.expect('FT.AGGREGATE', 'json_idx', '*', 'LOAD', '3', '@$.subj1', 'AS', 'a', 'APPLY', '(@a+@a)/2', 'AS', 'avg') \
.equal([1, ['a', '3.14', 'avg', '3.14']])
@skip(msan=True, no_json=True)
def test_MOD1266(env):
# Test parsing failure
conn = getConnectionByEnv(env)
env.cmd('FT.CREATE', 'idx', 'SCHEMA', 'n1', 'NUMERIC', 'SORTABLE', 'n2', 'NUMERIC', 'SORTABLE')
conn.execute_command('HSET', 'doc1', 'n1', '1', 'n2', '1')
conn.execute_command('HSET', 'doc2', 'n1', '2', 'n2', '2')
conn.execute_command('HSET', 'doc2', 'n1', 'foo', 'n2', '-999')
conn.execute_command('HSET', 'doc3', 'n1', '3', 'n2', '3')
env.expect('FT.SEARCH', 'idx', '*', 'sortby', 'n2', 'DESC', 'RETURN', '1', 'n2') \
.equal([2, 'doc3', ['n2', '3'], 'doc1', ['n2', '1']])
assertInfoField(env, 'idx', 'num_docs', 2)
# Test fetching failure. An object cannot be indexed
env.cmd('FT.CREATE', 'jsonidx', 'ON', 'JSON', 'SCHEMA', '$.t', 'TEXT')
conn.execute_command('JSON.SET', '1', '$', r'{"t":"Redis"}')
env.expect('FT.SEARCH', 'jsonidx', '*').equal([1, '1', ['$', '{"t":"Redis"}']])
env.expect('FT.SEARCH', 'jsonidx', 'redis').equal([1, '1', ['$', '{"t":"Redis"}']])
conn.execute_command('JSON.SET', '1', '$.t', r'{"inner_t":"Redis"}')
env.expect('FT.SEARCH', 'jsonidx', '*').equal([0])
def testMemAllocated(env):
conn = getConnectionByEnv(env)
# sanity
env.cmd('FT.CREATE', 'idx1', 'SCHEMA', 't', 'TEXT')
assertInfoField(env, 'idx1', 'key_table_size_mb', '1.52587890625e-5', delta = 0.01)
conn.execute_command('HSET', 'doc1', 't', 'foo bar baz')
assertInfoField(env, 'idx1', 'key_table_size_mb', '2.765655517578125e-05', delta=0.01)
conn.execute_command('HSET', 'doc2', 't', 'hello world')
assertInfoField(env, 'idx1', 'key_table_size_mb', '8.296966552734375e-05', delta=0.01)
conn.execute_command('HSET', 'd3', 't', 'help')
assertInfoField(env, 'idx1', 'key_table_size_mb', '0.00013828277587890625', delta=0.01)
conn.execute_command('DEL', 'd3')
assertInfoField(env, 'idx1', 'key_table_size_mb', '8.296966552734375e-05', delta=0.01)
conn.execute_command('DEL', 'doc1')
assertInfoField(env, 'idx1', 'key_table_size_mb', '2.765655517578125e-05', delta=0.01)
conn.execute_command('DEL', 'doc2')
assertInfoField(env, 'idx1', 'key_table_size_mb', '1.52587890625e-5', delta = 0.01)
# mass
env.cmd('FT.CREATE', 'idx2', 'SCHEMA', 't', 'TEXT')
for i in range(1000):
conn.execute_command('HSET', f'doc{i}', 't', f'text{i}')
assertInfoField(env, 'idx2', 'key_table_size_mb', '0.027684211730957031', delta=0.01)
for i in range(1000):
conn.execute_command('DEL', f'doc{i}')
assertInfoField(env, 'idx2', 'key_table_size_mb', '1.52587890625e-5', delta = 0.01)
def testUNF(env):
conn = getConnectionByEnv(env)
env.cmd('FT.CREATE', 'idx', 'SCHEMA',
'txt', 'TEXT', 'SORTABLE',
'txt_unf', 'TEXT', 'SORTABLE', 'UNF',
'tag', 'TAG', 'SORTABLE',
'tag_unf', 'TAG', 'SORTABLE', 'UNF')
conn.execute_command('HSET', 'doc1', 'txt', 'FOO', 'txt_unf', 'FOO',
'tag', 'FOO', 'tag_unf', 'FOO')
# test `FOO`
env.expect('FT.AGGREGATE', 'idx', '*', 'GROUPBY', '4', '@txt', '@txt_unf', '@tag', '@tag_unf') \
.equal([1, ['txt', 'foo', 'txt_unf', 'FOO', 'tag', 'foo', 'tag_unf', 'FOO']])
# test `Maße`
conn.execute_command('HSET', 'doc1', 'txt', u'Maße', 'txt_unf', u'Maße',
'tag', u'Maße', 'tag_unf', u'Maße')
env.expect('FT.AGGREGATE', 'idx', '*', 'GROUPBY', '4', '@txt', '@txt_unf', '@tag', '@tag_unf') \
.equal([1, ['txt', 'masse', 'txt_unf', u'Maße', 'tag', 'masse', 'tag_unf', u'Maße']])
# test `Maße` with LOAD
conn.execute_command('HSET', 'doc1', 'txt', 'Maße', 'txt_unf', u'Maße',
'tag', 'Maße', 'tag_unf', u'Maße')
env.expect('FT.AGGREGATE', 'idx', '*', \
'LOAD', '4', '@txt', '@txt_unf', '@tag', '@tag_unf', \
'GROUPBY', '4', '@txt', '@txt_unf', '@tag', '@tag_unf') \
.equal([1, ['txt', u'Maße', 'txt_unf', u'Maße', 'tag', u'Maße', 'tag_unf', 'Maße']])
def test_MOD_1517(env):
conn = getConnectionByEnv(env)
env.cmd('FT.CREATE', 'idx', 'SCHEMA', 'field1', 'TAG', 'SORTABLE',
'field2', 'TAG', 'SORTABLE')
# both fields exist
conn.execute_command('HSET', 'doc1', 'field1', 'val1', 'field2', 'val2', 'amount1', '1', 'amount2', '1')
# first tag is nil
conn.execute_command('HSET', 'doc2', 'field2', 'val2', 'amount1', '1', 'amount2', '1')
# second tag is nil
conn.execute_command('HSET', 'doc3', 'field1', 'val1', 'amount1', '1', 'amount2', '1')
# both tags are nil
conn.execute_command('HSET', 'doc4', 'amount1', '1', 'amount2', '1')
res = [4, ['field1', None, 'field2', None, 'amount1Sum', '1', 'amount2Sum', '1'],
['field1', 'val1', 'field2', 'val2', 'amount1Sum', '1', 'amount2Sum', '1'],
['field1', None, 'field2', 'val2', 'amount1Sum', '1', 'amount2Sum', '1'],
['field1', 'val1', 'field2', None, 'amount1Sum', '1', 'amount2Sum', '1']]
actual = conn.execute_command('FT.AGGREGATE', 'idx', '*',
'LOAD', '2', '@amount1', '@amount2',
'GROUPBY', '2', '@field1', '@field2',
'REDUCE', 'SUM', '1', '@amount1', 'AS', 'amount1Sum',
'REDUCE', 'SUM', '1', '@amount2', 'as', 'amount2Sum')
# The order of the groups themselves is not guaranteed, so compare the group rows regardless of order.
env.assertEqual(actual[0], res[0])
env.assertEqual(sorted(actual[1:], key=str), sorted(res[1:], key=str))
@skip(msan=True, no_json=True)
def test_MOD1544(env):
# Test parsing failure
conn = getConnectionByEnv(env)
env.cmd('FT.CREATE', 'idx', 'ON', 'JSON', 'SCHEMA', '$.name', 'AS', 'name', 'TEXT')
conn.execute_command('JSON.SET', '1', '.', '{"name": "John Smith"}')
# res = [1, '1', ['name', 'John Smith']]
# Highlight/summarize is not supported with JSON indexes
error_msg = "HIGHLIGHT/SUMMARIZE is not supported with JSON indexes"
env.expect('FT.SEARCH', 'idx', '@name:(John)', 'RETURN', '1', 'name',
'HIGHLIGHT').error().contains(error_msg)
env.expect('FT.SEARCH', 'idx', '@name:(John)', 'RETURN', '1', 'name',
'HIGHLIGHT', 'FIELDS', '1', 'name').error().contains(error_msg)
def test_MOD_1808(env):
conn = getConnectionByEnv(env)
env.expect('FT.CREATE', 'idx', 'SCHEMA', 't', 'TEXT').ok()
conn.execute_command('hset', 'doc0', 't', 'world0')
conn.execute_command('hset', 'doc1', 't', 'world1')
conn.execute_command('hset', 'doc2', 't', 'world2')
conn.execute_command('hset', 'doc3', 't', 'world3')
res = env.cmd('FT.SEARCH', 'idx', '(~@t:world2) (~@t:world1) (~@t:wada)', 'SUMMARIZE', 'FRAGS', '1', 'LEN', '25', 'HIGHLIGHT', 'TAGS', "", '')
env.assertEqual(toSortedFlatList(res), toSortedFlatList([4, 'doc2', ['t', "world2... "], 'doc1', ['t', "world1... "], 'doc0', ['t', 'world0'], 'doc3', ['t', 'world3']]))
def test_2370(env):
# Test limit offset great than number of results
conn = getConnectionByEnv(env)
env.cmd('FT.CREATE', 'idx', 'SCHEMA', 't1', 'TEXT', 't2', 'TEXT')
conn.execute_command('HSET', 'doc1', 't1', 'baz')
conn.execute_command('HSET', 'doc2', 't1', 'foo', 't2', 'bar')
# number of results is lower than LIMIT
env.expect('FT.SEARCH', 'idx', '*', 'LIMIT', '10', '10').equal([2])
# missing fields
env.expect('FT.SEARCH', 'idx', '*').equal([2, 'doc1', ['t1', 'baz'], 'doc2', ['t1', 'foo', 't2', 'bar']])
def test_MOD1907(env):
# Test FT.CREATE w/o fields parameters
env.expect('FT.CREATE', 'idx', 'ON', 'HASH', 'SCHEMA').error().contains('Fields arguments are missing')
env.expect('FT.CREATE', 'idx', 'STOPWORDS', 0, 'SCHEMA').error().contains('Fields arguments are missing')
@skip(cluster=True)
def test_SkipFieldWithNoMatch(env):
conn = getConnectionByEnv(env)
env.cmd(config_cmd(), 'SET', '_PRINT_PROFILE_CLOCK', 'false')
env.cmd('FT.CREATE', 'idx', 'SCHEMA', 't1', 'TEXT', 't2', 'TEXT')
conn.execute_command('HSET', 'doc1', 't1', 'foo', 't2', 'bar')
res = env.cmd('FT.PROFILE', 'idx', 'SEARCH', 'QUERY', '@t1:foo')
env.assertEqual(res[1][1][0][3], ['Type', 'TEXT', 'Term', 'foo', 'Number of reading operations', 1, 'Estimated number of matches', 1])
res = env.cmd('FT.PROFILE', 'idx', 'SEARCH', 'QUERY', 'foo')
env.assertEqual(res[1][1][0][3], ['Type', 'TEXT', 'Term', 'foo', 'Number of reading operations', 1, 'Estimated number of matches', 1])
# bar exists in `t2` only
res = env.cmd('FT.PROFILE', 'idx', 'SEARCH', 'QUERY', '@t1:bar')
env.assertEqual(res[1][1][0][3], ['Type', 'EMPTY', 'Number of reading operations', 0])
res = env.cmd('FT.PROFILE', 'idx', 'SEARCH', 'QUERY', 'bar')
env.assertEqual(res[1][1][0][3], ['Type', 'TEXT', 'Term', 'bar', 'Number of reading operations', 1, 'Estimated number of matches', 1] )
# Check with NOFIELDS flag
env.cmd('FT.CREATE', 'idx_nomask', 'NOFIELDS', 'SCHEMA', 't1', 'TEXT', 't2', 'TEXT')
waitForIndex(env, 'idx_nomask')
res = env.cmd('FT.PROFILE', 'idx_nomask', 'SEARCH', 'QUERY', '@t1:foo')
env.assertEqual(res[1][1][0][3], ['Type', 'TEXT', 'Term', 'foo', 'Number of reading operations', 1, 'Estimated number of matches', 1])
res = env.cmd('FT.PROFILE', 'idx_nomask', 'SEARCH', 'QUERY', 'foo')
env.assertEqual(res[1][1][0][3], ['Type', 'TEXT', 'Term', 'foo', 'Number of reading operations', 1, 'Estimated number of matches', 1])
res = env.cmd('FT.PROFILE', 'idx_nomask', 'SEARCH', 'QUERY', '@t1:bar')
env.assertEqual(res[1][1][0][3], ['Type', 'TEXT', 'Term', 'bar', 'Number of reading operations', 1, 'Estimated number of matches', 1])
res = env.cmd('FT.PROFILE', 'idx_nomask', 'SEARCH', 'QUERY', 'bar')
env.assertEqual(res[1][1][0][3], ['Type', 'TEXT', 'Term', 'bar', 'Number of reading operations', 1, 'Estimated number of matches', 1])
@skip(cluster=True)
def test_update_num_terms(env):
conn = getConnectionByEnv(env)
env.cmd(config_cmd(), 'SET', 'FORK_GC_CLEAN_THRESHOLD', '0')
env.cmd('FT.CREATE', 'idx', 'SCHEMA', 't', 'TEXT')
conn.execute_command('HSET', 'doc1', 't', 'foo')
conn.execute_command('HSET', 'doc1', 't', 'bar')
assertInfoField(env, 'idx', 'num_terms', 2)
forceInvokeGC(env, 'idx')
assertInfoField(env, 'idx', 'num_terms', 1)
@skip(cluster=True)
def testOverMaxResults():
env = Env(moduleArgs='MAXSEARCHRESULTS 20')
conn = getConnectionByEnv(env)
commands = [
[config_cmd(), 'SET', 'MAXAGGREGATERESULTS', '25'],
[config_cmd(), 'SET', 'MAXAGGREGATERESULTS', '20'],
[config_cmd(), 'SET', 'MAXAGGREGATERESULTS', '15'],
]
for c in commands:
env.cmd(*c)
env.cmd('flushall')
env.cmd('FT.CREATE', 'idx', 'SCHEMA', 't', 'TEXT')
# test with number of documents lesser than MAXSEARCHRESULTS
for i in range(10):
conn.execute_command('HSET', i, 't', i)
res = [10, '0', '1', '2', '3', '4', '5', '6', '7', '8', '9']
env.expect('FT.SEARCH', 'idx', '*', 'NOCONTENT').equal(res)
env.expect('FT.SEARCH', 'idx', '*', 'NOCONTENT', 'LIMIT', '0', '10').equal(res)
env.expect('FT.SEARCH', 'idx', '*', 'NOCONTENT', 'LIMIT', '1', '20').equal([res[0], *res[2:]])
env.expect('FT.SEARCH', 'idx', '*', 'NOCONTENT', 'LIMIT', '5', '10').equal([res[0], *res[6:11]])
env.expect('FT.SEARCH', 'idx', '*', 'NOCONTENT', 'LIMIT', '10', '10').equal([10])
env.expect('FT.SEARCH', 'idx', '*', 'NOCONTENT', 'LIMIT', '20', '10').equal([10])
env.expect('FT.SEARCH', 'idx', '*', 'NOCONTENT', 'LIMIT', '30', '10').equal('SEARCH_LIMIT_OVER OFFSET exceeds maximum of 20')
# test with number of documents equal to MAXSEARCHRESULTS
for i in range(10,20):
conn.execute_command('HSET', i, 't', i)
res = [20, '10', '11', '12', '13', '14', '15', '16', '17', '18', '19']
env.expect('FT.SEARCH', 'idx', '*', 'NOCONTENT', 'LIMIT', '10', '10').equal(res)
env.expect('FT.SEARCH', 'idx', '*', 'NOCONTENT', 'LIMIT', '1', '20').equal([res[0], *[str(i) for i in range(1, 20, 1)]])
env.expect('FT.SEARCH', 'idx', '*', 'NOCONTENT', 'LIMIT', '15', '10').equal([20, *res[6:11]])
env.expect('FT.SEARCH', 'idx', '*', 'NOCONTENT', 'LIMIT', '20', '10').equal([20])
env.expect('FT.SEARCH', 'idx', '*', 'NOCONTENT', 'LIMIT', '30', '10').equal('SEARCH_LIMIT_OVER OFFSET exceeds maximum of 20')
# test with number of documents greater than MAXSEARCHRESULTS
for i in range(20,30):
conn.execute_command('HSET', i, 't', i)
env.expect('FT.SEARCH', 'idx', '*', 'NOCONTENT', 'LIMIT', '1', '20').equal([30, *[str(i) for i in range(1, 20, 1)]])
env.expect('FT.SEARCH', 'idx', '*', 'NOCONTENT', 'LIMIT', '10', '10').equal([30, *res[1:11]])
env.expect('FT.SEARCH', 'idx', '*', 'NOCONTENT', 'LIMIT', '15', '10').equal([30, *res[6:11]])
env.expect('FT.SEARCH', 'idx', '*', 'NOCONTENT', 'LIMIT', '20', '10').equal([30])
env.expect('FT.SEARCH', 'idx', '*', 'NOCONTENT', 'LIMIT', '25', '10').equal('SEARCH_LIMIT_OVER OFFSET exceeds maximum of 20')
env.expect('FT.SEARCH', 'idx', '*', 'NOCONTENT', 'LIMIT', '30', '10').equal('SEARCH_LIMIT_OVER OFFSET exceeds maximum of 20')
def test_MOD_3372(env):
conn = getConnectionByEnv(env)
conn.execute_command('FT.CREATE', 'idx', 'SCHEMA', 't', 'TEXT')
env.expect('FT.EXPLAIN').error().contains('wrong number of arguments')
env.expect('FT.EXPLAIN', 'idx').error().contains('wrong number of arguments')
env.expect('FT.EXPLAIN', 'idx', 'foo').equal('UNION {\n foo\n +foo(expanded)\n}\n')
env.expect('FT.EXPLAIN', 'idx', 'foo', 'verbatim').equal('foo\n')
env.expect('FT.EXPLAIN', 'non-exist', 'foo').error().equal('SEARCH_INDEX_NOT_FOUND Index not found: non-exist')
if not env.isCluster():
# FT.EXPLAINCLI is not supported by the coordinator
env.expect('FT.EXPLAINCLI').error().contains('wrong number of arguments')
env.expect('FT.EXPLAINCLI', 'idx').error().contains('wrong number of arguments')
env.expect('FT.EXPLAINCLI', 'idx', 'foo').equal(['UNION {', ' foo', ' +foo(expanded)', '}', ''])
env.expect('FT.EXPLAINCLI', 'idx', 'foo', 'verbatim').equal(['foo', ''])
env.expect('FT.EXPLAINCLI', 'non-exist', 'foo').error().equal('SEARCH_INDEX_NOT_FOUND Index not found: non-exist')
def test_MOD_3540(env):
# disable SORTBY MAX for FT.SEARCH
conn = getConnectionByEnv(env)
conn.execute_command('FT.CREATE', 'idx', 'SCHEMA', 't', 'TEXT')
for i in range(10):
conn.execute_command('HSET', i, 't', i)
env.expect('FT.SEARCH', 'idx', '*', 'SORTBY', 't', 'DESC', 'MAX', '1').error() \
.contains('SORTBY MAX is not supported by FT.SEARCH')
env.expect('FT.AGGREGATE', 'idx', '*', 'SORTBY', '2', '@t', 'DESC', 'MAX', '1', 'LOAD', '*') \
.equal([10, ['t', '9']])
# SORTBY MAX followed by LIMIT
env.expect('FT.AGGREGATE', 'idx', '*', 'SORTBY', '2', '@t', 'DESC', 'MAX', '1', 'LIMIT', '0', '2', 'LOAD', '*') \
.equal([10, ['t', '9'], ['t', '8']])
env.expect('FT.AGGREGATE', 'idx', '*', 'SORTBY', '2', '@t', 'DESC', 'MAX', '2', 'LIMIT', '0', '1', 'LOAD', '*') \
.equal([10, ['t', '9']])
env.expect('FT.AGGREGATE', 'idx', '*', 'SORTBY', '2', '@t', 'DESC', 'MAX', '1', 'LIMIT', '0', '0', 'LOAD', '*') \
.equal([10])
env.expect('FT.AGGREGATE', 'idx', '*', 'SORTBY', '2', '@t', 'DESC', 'MAX', '0', 'LIMIT', '0', '1', 'LOAD', '*') \
.equal([10, ['t', '9']])
# LIMIT followed by SORTBY MAX
env.expect('FT.AGGREGATE', 'idx', '*', 'LIMIT', '0', '2', 'SORTBY', '2', '@t', 'DESC', 'MAX', '1', 'LOAD', '*') \
.equal([10, ['t', '9']])
env.expect('FT.AGGREGATE', 'idx', '*', 'LIMIT', '0', '1', 'SORTBY', '2', '@t', 'DESC', 'MAX', '2', 'LOAD', '*') \
.equal([10, ['t', '9'], ['t', '8']])
env.expect('FT.AGGREGATE', 'idx', '*', 'LIMIT', '0', '1', 'SORTBY', '2', '@t', 'DESC', 'MAX', '0', 'LOAD', '*') \
.equal([10, ['t', '9'], ['t', '8'], ['t', '7'], ['t', '6'], ['t', '5'], ['t', '4'], ['t', '3'], ['t', '2'], ['t', '1'], ['t', '0']])
env.expect('FT.AGGREGATE', 'idx', '*', 'LIMIT', '0', '0', 'SORTBY', '2', '@t', 'DESC', 'MAX', '1', 'LOAD', '*') \
.equal([10])
def test_sortby_Noexist(env):
conn = getConnectionByEnv(env)
env.cmd('FT.CREATE', 'idx', 'SCHEMA', 't', 'TEXT')
conn.execute_command('HSET', 'doc1', 't', '1')
conn.execute_command('HSET', 'doc2', 'somethingelse', '2')
conn.execute_command('HSET', 'doc3', 't', '3')
conn.execute_command('HSET', 'doc4', 'somethingelse', '4')
env.expect('FT.SEARCH', 'idx', '*', 'SORTBY', 't', 'ASC', 'LIMIT', '0', '2').equal([4, 'doc1', ['t', '1'], 'doc3', ['t', '3']])
env.expect('FT.SEARCH', 'idx', '*', 'SORTBY', 't', 'DESC', 'LIMIT', '0', '2').equal([4, 'doc3', ['t', '3'], 'doc1', ['t', '1']])
# receive a result w/o sortby field at the end.
# remove in test to support test on cluster
res = env.cmd('FT.SEARCH', 'idx', '*', 'SORTBY', 't', 'ASC', 'LIMIT', '0', '3')
env.assertEqual(res[0:5], [4, 'doc1', ['t', '1'], 'doc3', ['t', '3']])
res = env.cmd('FT.SEARCH', 'idx', '*', 'SORTBY', 't', 'DESC', 'LIMIT', '0', '3')
env.assertEqual(res[0:5], [4, 'doc3', ['t', '3'], 'doc1', ['t', '1']])
if not env.isCluster():
env.expect('FT.SEARCH', 'idx', '*', 'SORTBY', 't', 'ASC', 'LIMIT', '0', '3').equal([4, 'doc1', ['t', '1'], 'doc3', ['t', '3'], 'doc2', ['somethingelse', '2']])
env.expect('FT.SEARCH', 'idx', '*', 'SORTBY', 't', 'DESC', 'LIMIT', '0', '3').equal([4, 'doc3', ['t', '3'], 'doc1', ['t', '1'], 'doc4', ['somethingelse', '4']])
def test_sortby_Noexist_Sortables(env):
''' issue 3457 '''
conn = getConnectionByEnv(env)
sortable_options = [[True,True], [True,False], [False,True], [False,False]]
for count, args in enumerate(sortable_options):
sortable1 = ['SORTABLE'] if args[0] else []
sortable2 = ['SORTABLE'] if args[1] else []
conn.execute_command('FT.CREATE', f'idx{count}', 'SCHEMA', 'numval', 'NUMERIC' , *sortable1,
'text', 'TEXT', *sortable2)
for count, args in enumerate(sortable_options):
# Use cluster {hashtag} to handle which keys are on the same shard (same cluster slot)
conn.execute_command('HSET', '{key1}1', 'numval', '110')
conn.execute_command('HSET', '{key1}2', 'numval', '108')
conn.execute_command('HSET', '{key2}1', 'text', 'Meow')
conn.execute_command('HSET', '{key2}2', 'text', 'Chirp')
msg = f'sortable1: {sortable1}, sortable2: {sortable2}'
# Check ordering of docs:
# In cluster: Docs without sortby field are ordered by key name
# In non-cluster: Docs without sortby field are ordered by doc id (order of insertion/update)
res = conn.execute_command('FT.SEARCH', f'idx{count}', '*', 'sortby', 'numval', 'ASC')
env.assertEqual(res, [4,
'{key1}2', ['numval', '108'], '{key1}1', ['numval', '110'],
'{key2}1', ['text', 'Meow'], '{key2}2', ['text', 'Chirp'],
], message=msg)
res = conn.execute_command('FT.SEARCH', f'idx{count}', '*', 'sortby', 'numval', 'DESC')
env.assertEqual(res, [4,
'{key1}1', ['numval', '110'], '{key1}2', ['numval', '108'],
'{key2}2', ['text', 'Chirp'], '{key2}1', ['text', 'Meow'],
], message=msg)
# Add more keys
conn.execute_command('HSET', '{key1}3', 'text', 'Bark')
conn.execute_command('HSET', '{key1}4', 'text', 'Quack')
conn.execute_command('HSET', '{key2}3', 'numval', '109')
conn.execute_command('HSET', '{key2}4', 'numval', '111')
conn.execute_command('HSET', '{key2}5', 'numval', '108')
conn.execute_command('HSET', '{key2}6', 'text', 'Squeak')
for count, args in enumerate(sortable_options):
res = conn.execute_command('FT.SEARCH', f'idx{count}', '*', 'sortby', 'numval', 'ASC')
if env.isCluster():
env.assertEqual(res, [10,
'{key1}2', ['numval', '108'],
'{key2}5', ['numval', '108'],
'{key2}3', ['numval', '109'],
'{key1}1', ['numval', '110'],
'{key2}4', ['numval', '111'],
'{key1}3', ['text', 'Bark'],
'{key1}4', ['text', 'Quack'],
'{key2}1', ['text', 'Meow'],
'{key2}2', ['text', 'Chirp'],
'{key2}6', ['text', 'Squeak'],
], message=msg)
else:
env.assertEqual(res, [10,
'{key1}2', ['numval', '108'],
'{key2}5', ['numval', '108'],
'{key2}3', ['numval', '109'],
'{key1}1', ['numval', '110'],
'{key2}4', ['numval', '111'],
'{key2}1', ['text', 'Meow'],
'{key2}2', ['text', 'Chirp'],
'{key1}3', ['text', 'Bark'],
'{key1}4', ['text', 'Quack'],
'{key2}6', ['text', 'Squeak'],
], message=msg)
res = conn.execute_command('FT.SEARCH', f'idx{count}', '*', 'sortby', 'numval', 'DESC')
if env.isCluster():
env.assertEqual(res, [10,
'{key2}4', ['numval', '111'],
'{key1}1', ['numval', '110'],
'{key2}3', ['numval', '109'],
'{key2}5', ['numval', '108'],
'{key1}2', ['numval', '108'],
'{key2}6', ['text', 'Squeak'],
'{key2}2', ['text', 'Chirp'],
'{key2}1', ['text', 'Meow'],
'{key1}4', ['text', 'Quack'],
'{key1}3', ['text', 'Bark'],
], message=msg)
else:
env.assertEqual(res, [10,
'{key2}4', ['numval', '111'],
'{key1}1', ['numval', '110'],
'{key2}3', ['numval', '109'],
'{key2}5', ['numval', '108'],
'{key1}2', ['numval', '108'],
'{key2}6', ['text', 'Squeak'],
'{key1}4', ['text', 'Quack'],
'{key1}3', ['text', 'Bark'],
'{key2}2', ['text', 'Chirp'],
'{key2}1', ['text', 'Meow'],
], message=msg)
def testDeleteIndexes(env):
# test cleaning of all specs from a prefix
conn = getConnectionByEnv(env)
for i in range(10):
env.cmd('FT.CREATE', i, 'PREFIX', '1', i / 2, 'SCHEMA', 't', 'TEXT')
env.cmd('FT.DROPINDEX', i)
# create an additional index
env.cmd('FT.CREATE', i, 'PREFIX', '1', i / 2, 'SCHEMA', 't', 'TEXT')
def test_mod_4207(env):
conn = getConnectionByEnv(env)
env.cmd('FT.CREATE', 'idx1', 'FILTER', 'EXISTS(@country)', 'SCHEMA', 'business', 'TEXT', 'country', 'TEXT')
env.cmd('FT.CREATE', 'idx2', 'FILTER', 'EXISTS(@business)', 'SCHEMA', 'business', 'TEXT', 'country', 'TEXT')
conn.execute_command('HSET', 'address:1', 'business', 'bar', 'country', 'USA')
conn.execute_command('HSET', 'address:2', 'business', 'baz', 'country', 'Israel')
conn.execute_command('HSET', 'address:3', 'business', 'foo')
conn.execute_command('HSET', 'address:4', 'country', 'Israel')
env.expect('FT.SEARCH', 'idx1', '*', 'NOCONTENT', 'SORTBY', 'business').equal([3, 'address:1', 'address:2', 'address:4'])
env.expect('FT.SEARCH', 'idx2', '*', 'NOCONTENT', 'SORTBY', 'business').equal([3, 'address:1', 'address:2', 'address:3'])
@skip(cluster=True)
def test_mod_4255(env):
conn = getConnectionByEnv(env)
env.expect('FT.CREATE', 'idx', 'ON', 'HASH', 'SCHEMA', 'test', 'TEXT').equal('OK')
conn.execute_command('HSET', 'doc1', 'test', '1')
conn.execute_command('HSET', 'doc2', 'test', '2')
# test normal case
# get first result
res = env.cmd('FT.AGGREGATE', 'idx', '*', 'LOAD', '1', '@test', 'WITHCURSOR', 'COUNT', '1')
env.assertEqual(res[0] ,[1, ['test', '1']])
cursor = res[1]
env.assertNotEqual(cursor ,0)
# get second result
res = env.cmd('FT.CURSOR', 'READ', 'idx', cursor)
env.assertEqual(res[0] ,[1, ['test', '2']])
cursor = res[1]
env.assertNotEqual(cursor ,0)
# get empty results after cursor was exhausted
env.expect('FT.CURSOR', 'READ', 'idx', cursor).equal([[0], 0])
# Test cursor after data structure that has changed due to insert
res = env.cmd('FT.AGGREGATE', 'idx', '*', 'LOAD', '1', '@test', 'WITHCURSOR', 'COUNT', '1')
cursor = res[1]
for i in range(3, 1001, 1):
conn.execute_command('HSET', f'doc{i}', 'test', str(i))
res = env.cmd('FT.CURSOR', 'READ', 'idx', cursor)
env.assertEqual(res[0] ,[1, ['test', '2']])
env.assertNotEqual(cursor ,0)
# Test cursor after data structure that has changed due to insert
res = env.cmd('FT.AGGREGATE', 'idx', '*', 'LOAD', '1', '@test', 'WITHCURSOR', 'COUNT', '1')
env.assertEqual(res[0] ,[1, ['test', '1']])
cursor = res[1]
env.assertNotEqual(cursor ,0)
for i in range(3, 1001, 1):
conn.execute_command('DEL', f'doc{i}', 'test', str(i))
forceInvokeGC(env, 'idx')
res = env.cmd('FT.CURSOR', 'READ', 'idx', cursor)
env.assertEqual(res[0] ,[1, ['test', '2']])
cursor = res[1]
env.assertNotEqual(cursor ,0)
res = env.cmd('FT.CURSOR', 'READ', 'idx', cursor)
cursor = res[1]
env.assertEqual(cursor ,0)
@skip(no_json=True)
def test_as_startswith_as(env):
conn = getConnectionByEnv(env)
env.expect('FT.CREATE', 'idx', 'ON', 'JSON', 'SCHEMA', '$.attr1', 'AS', 'asa', 'TEXT').equal('OK')
conn.execute_command('JSON.SET', 'doc2', '$', '{"attr1": "foo", "attr2": "bar"}')
env.expect('FT.SEARCH', 'idx', '@asa:(foo)', 'RETURN', 1, 'asa').equal([1, 'doc2', ['asa', 'foo']])
env.expect('FT.SEARCH', 'idx', '@asa:(foo)', 'RETURN', 3, 'asa', 'AS', 'asa').equal([1, 'doc2', ['asa', 'foo']])
env.expect('FT.SEARCH', 'idx', '@asa:(foo)', 'RETURN', 3, '$.attr1', 'AS', 'asa').equal([1, 'doc2', ['asa', 'foo']])
env.expect('FT.SEARCH', 'idx', '@asa:(foo)', 'RETURN', 3, '$.attr1', 'AS', '$.attr2').equal([1, 'doc2', ['$.attr2', 'foo']])
env.expect('FT.SEARCH', 'idx', '@asa:(foo)', 'RETURN', 3, 'asa', 'AS', '$.attr2').equal([1, 'doc2', ['$.attr2', 'foo']])
env.expect('FT.AGGREGATE', 'idx', '@asa:(foo)', 'LOAD', 1, 'asa').equal([1, ['asa', 'foo']])
env.expect('FT.AGGREGATE', 'idx', '@asa:(foo)', 'LOAD', 3, 'asa', 'AS', 'asa').equal([1, ['asa', 'foo']])
env.expect('FT.AGGREGATE', 'idx', '@asa:(foo)', 'LOAD', 3, '$.attr1', 'AS', 'asa').equal([1, ['asa', 'foo']])
env.expect('FT.AGGREGATE', 'idx', '@asa:(foo)', 'LOAD', 3, '$.attr1', 'AS', '$.attr2').equal([1, ['$.attr2', 'foo']])
env.expect('FT.AGGREGATE', 'idx', '@asa:(foo)', 'LOAD', 3, 'asa', 'AS', '$.attr2').equal([1, ['$.attr2', 'foo']])
def test_mod4296_badexpr(env):
env.expect('FT.CREATE', 'idx', 'SCHEMA', 't', 'TEXT').equal('OK')
env.expect('HSET', 'doc', 't', 'foo').equal(1)
env.expect('FT.AGGREGATE', 'idx', 'foo', 'LOAD', 1, '@t', 'APPLY', '1%0', 'as', 'foo').apply(lambda x: str(float(x[-1][-1]))).equal('nan')
env.expect('FT.AGGREGATE', 'idx', 'foo', 'LOAD', 1, '@t', 'APPLY', '1/0', 'as', 'foo').equal([1, ['t', 'foo', 'foo', 'inf']])
@skip(cluster=True)
def test_mod5062(env):
env.expect(config_cmd(), 'SET', 'MAXSEARCHRESULTS', '0').ok()
env.expect(config_cmd(), 'SET', 'MAXAGGREGATERESULTS', '0').ok()
n = 100
env.expect('FT.CREATE', 'idx', 'ON', 'HASH', 'SCHEMA', 't', 'TEXT').ok()
for i in range(n):
env.expect('HSET', i, 't', 'hello world').equal(1)
# verify no crash
env.expect('FT.SEARCH', 'idx', 'hello').equal([n])
# verify using counter instead of sorter
search_profile = env.cmd('FT.PROFILE', 'idx', 'SEARCH', 'QUERY', 'hello')
rp_profile = to_dict(search_profile[1][1][0])['Result processors profile']
env.assertEqual('Counter', to_dict(rp_profile[2])['Type'])
# verify no crash
env.expect('FT.AGGREGATE', 'idx', 'hello').noError()
env.expect('FT.AGGREGATE', 'idx', 'hello', 'LIMIT', 0, 0).equal([n])
# verify using counter instead of sorter, even with explicit sort
aggregate_profile = env.cmd('FT.PROFILE', 'idx', 'AGGREGATE', 'QUERY', 'hello', 'SORTBY', '1', '@t')
rp_profile = to_dict(aggregate_profile[1][1][0])['Result processors profile']
env.assertEqual('Counter', to_dict(rp_profile[1])['Type'])
def test_mod5252(env):
# Create an index and add a document
env.expect('FT.CREATE', 'idx', 'SCHEMA', 't', 'TEXT', 'n', 'NUMERIC').equal('OK')
env.expect('HSET', 'doc', 't', 'Hello', 'n', '1').equal(2)
# Test that the document is returned with the key name on a search command
res = env.cmd('FT.SEARCH', 'idx', '*', 'RETURN', '1', '__key')
env.assertEqual(res, [1, 'doc', ['__key', 'doc']])
# Test that the document is returned with the key name WITH ALIAS on a search command
res = env.cmd('FT.SEARCH', 'idx', '*', 'RETURN', '3', '__key', 'AS', 'key_name')
env.assertEqual(res, [1, 'doc', ['key_name', 'doc']])
# Test that the document is returned with the key name on an aggregate command
res = env.cmd('FT.AGGREGATE', 'idx', '*', 'LOAD', '1', '@__key', 'SORTBY', '1', '@__key')
env.assertEqual(res, [1, ['__key', 'doc']])
# Test that the document is returned with the key name WITH ALIAS on an aggregate command
res = env.cmd('FT.AGGREGATE', 'idx', '*', 'LOAD', '3', '@__key', 'AS', 'key_name', 'SORTBY', '1', '@key_name')
env.assertEqual(res, [1, ['key_name', 'doc']])
@skip(cluster=True)
def test_mod_6276(env):
# Setting the gc threshold to 0 so the gc won't skip its periodic run
env.expect(config_cmd(), 'SET', 'FORK_GC_CLEAN_THRESHOLD', '0').ok()
# Create an index and add a document + garbage
env.expect('FT.CREATE', 'idx', 'SCHEMA', 't', 'TEXT').ok()
env.expect('HSET', 'doc', 't', 'Hello').equal(1)
# Actual Test
env.expect(debug_cmd(), 'GC_STOP_SCHEDULE', 'idx').ok() # Stop the gc from running uncontrollably
env.expect(debug_cmd(), 'GC_WAIT_FOR_JOBS').equal('DONE') # Make sure there are no running gc jobs
env.expect('MULTI').ok() # Start an atomic transaction:
env.cmd(debug_cmd(), 'GC_CONTINUE_SCHEDULE', 'idx') # 1. Reschedule the gc - add a job to the queue
env.cmd('FT.DROPINDEX', 'idx') # 2. Drop the index while the gc is running/queued
env.expect('EXEC').equal(['OK', 'OK']) # Execute the transaction
env.expect(debug_cmd(), 'GC_WAIT_FOR_JOBS').equal('DONE') # Wait for the gc to finish
def test_mod5791(env):
con = getConnectionByEnv(env)
env.expect('FT.CREATE', 'idx', 'SCHEMA', 't', 'TEXT', 'v', 'VECTOR', 'FLAT', 6, 'TYPE', 'FLOAT32', 'DISTANCE_METRIC', 'L2',
'DIM', 2).equal('OK')
env.assertEqual(2, con.execute_command('HSET', 'doc1', 't', 'Hello world', 'v', 'abcdefgh'))
env.assertEqual(2, con.execute_command('HSET', 'doc2', 't', 'Hello world', 'v', 'abcdefgi'))
# The RSIndexResult object should be constructed as following:
# UNION:
# INTERSECTION:
# metric
# term
# metric
# While computing the scores, RSIndexResult_IterateOffsets is called. Validate that there is no corruption when
# iterating the metric RSIndexResult (before, we treated it as "default" - which is the aggregate type, and we might
# try access non-existing fields).
res = env.cmd('FT.SEARCH', 'idx', '(@v:[VECTOR_RANGE 0.8 $blob] @t:hello) | @v:[VECTOR_RANGE 0.8 $blob]',
'WITHSCORES', 'DIALECT', '2', 'params', '2', 'blob', 'abcdefgh')
env.assertEqual(res[:2], [1, 'doc1'])
@skip(asan=True, cluster=False)
def test_mod5778_add_new_shard_to_cluster(env):
mod5778_add_new_shard_to_cluster(env)
@skip(asan=True, cluster=False)
def test_mod5778_add_new_shard_to_cluster_TLS():
cert_file, key_file, ca_cert_file, passphrase = get_TLS_args()
env = Env(useTLS=True, tlsCertFile=cert_file, tlsKeyFile=key_file, tlsCaCertFile=ca_cert_file, tlsPassphrase=passphrase)
mod5778_add_new_shard_to_cluster(env)
def mod5778_add_new_shard_to_cluster(env: Env):
for i in range(env.shardsCount):
verify_shard_init(env.getConnection(i))
conn = env.getConnection()
initial_shards_count = env.shardsCount
# The first two fields in the cluster info reply are the number of partition in thr cluster.
env.assertEqual(conn.execute_command("search.clusterinfo")[:2], ['num_partitions', int(initial_shards_count)])
# Add a new shard to the cluster. Internally we call CLUSTER MEET to connect the new shard
# to the cluster. Also, we internally wait for the cluster to be ready and call "search.CLUSTERREFRESH"
# and update the topology change in the new shard (this is where we had a crash in MOD-5778).
env.addShardToClusterIfExists()
new_shard_conn = env.getConnection(shardId=initial_shards_count+1)
verify_shard_init(new_shard_conn)
# Expect that the cluster will be aware of the new shard, but for redisearch coordinator, the new shard isn't
# considered part of the partition yet as it does not contain any slots.
env.assertEqual(int(new_shard_conn.execute_command("cluster info")['cluster_known_nodes']), initial_shards_count+1)
env.assertEqual(new_shard_conn.execute_command("search.clusterinfo")[:2], ['num_partitions', int(initial_shards_count)])
# Move one slot (0) to the new shard (according to https://redis.io/commands/cluster-setslot/)
new_shard_id = new_shard_conn.execute_command('CLUSTER MYID')
source_shard_id = conn.execute_command('CLUSTER MYID')
env.assertOk(new_shard_conn.execute_command(f"CLUSTER SETSLOT 0 IMPORTING {source_shard_id}"))
env.assertOk(conn.execute_command(f"CLUSTER SETSLOT 0 MIGRATING {new_shard_id}"))
env.assertOk(new_shard_conn.execute_command(f"CLUSTER SETSLOT 0 NODE {new_shard_id}"))
env.assertOk(conn.execute_command(f"CLUSTER SETSLOT 0 NODE {new_shard_id}"))
# Now we expect that the new shard will be a part of the cluster partition in redisearch (allow some time
# for the cluster refresh to occur and acknowledged by all shards)
env.waitCluster()
# search.clusterinfo response format is the following:
# ['num_partitions', 4, 'cluster_type', 'redis_oss', 'shards', [
# ['slots', [1, 5461], 'id', '60cdcb85a8f73f87ac6cc831ee799b75752aace3', 'host', '127.0.0.1', 'port', 6379],
# ['slots', [5462, 10923], 'id', '6b2af643a4d6f1723ff2b18b45216d1e0dc7befa', 'host', '127.0.0.1', 'port', 6381],
# ['slots', [10924, 16383], 'id', '4e51033405651441a4be6ddfb46cd85d0c54af6f', 'host', '127.0.0.1', 'port', 6383],
# ['slots', [0, 0], 'id', '1f834c5c207bbe8d6dab0c6f050ff06292eb333c', 'host', '127.0.0.1', 'port', 6385],
# ]]
cluster_info = new_shard_conn.execute_command("search.clusterinfo")
shards_idx = cluster_info.index('shards') + 1
unique_shards = set(shard[3] for shard in cluster_info[shards_idx])
with TimeLimit(10, f"Failed waiting for new shard to appear in search.clusterinfo: {cluster_info}"):
while len(unique_shards) != initial_shards_count+1:
time.sleep(0.1)
cluster_info = new_shard_conn.execute_command("search.clusterinfo")
shards_idx = cluster_info.index('shards') + 1
unique_shards = set(shard[3] for shard in cluster_info[shards_idx])
@skip(cluster=True)
def test_mod5910(env):
con = getConnectionByEnv(env)
env.expect('FT.CREATE', 'idx', 'SCHEMA', 't', 'TEXT', 'n', 'NUMERIC').equal('OK')
env.assertEqual(2, con.execute_command('HSET', 'doc1{tag}', 't', 'one', 'n', '1'))
env.assertEqual(2, con.execute_command('HSET', 'doc2{tag}', 't', 'two', 'n', '2'))
env.assertEqual(2, con.execute_command('HSET', 'doc3{tag}', 't', 'three', 'n', '3'))
# In this test, we run the following query twice. The query consists of an intersection between two sub-queries,
# where the number of expected results from the first sub-query (@n:[1 3]) is 3, while the number of expected
# results from the second subquery (@t:one | @t:two) is 2. Intersection iterator is sorting its children, so that
# children with lower number of expected results come first, to reduce the number of overall "skip_to" done by the
# iterator.
# Hence, we expect that the numeric iterator would come *after* the union iterator.
res = env.execute_command('FT.PROFILE', 'idx', 'search', 'query', '(@n:[1 3] (@t:one | @t:two))')
iterators_profile = to_dict(res[1][1][0])['Iterators profile']
env.assertEqual(iterators_profile[1], 'INTERSECT')
env.assertEqual(iterators_profile[7][0][1], 'UNION')
env.assertEqual(iterators_profile[7][1][1], 'NUMERIC')
# When _PRIORITIZE_INTERSECT_UNION_CHILDREN config is set, the number of expected results from the union iterator
# child is factored by its own number of children. Hence, the weighted expected number of results for the second
# sub-query evaluated in this case to 2*2=4 under this config, so now we expect that the numeric iterator would come
# *before* the union iterator.
env.assertEqual('OK', con.execute_command(config_cmd(), 'SET', '_PRIORITIZE_INTERSECT_UNION_CHILDREN', 'true'))
res = con.execute_command('FT.PROFILE', 'idx', 'search', 'query', '(@n:[1 3] (@t:one | @t:two))')
iterators_profile = to_dict(res[1][1][0])['Iterators profile']
env.assertEqual(iterators_profile[1], 'INTERSECT')
env.assertEqual(iterators_profile[7][0][1], 'NUMERIC')
env.assertEqual(iterators_profile[7][1][1], 'UNION')
@skip(cluster=True)
def test_mod5880(env):
env.cmd(config_cmd(), "set", "FORK_GC_CLEAN_THRESHOLD", "0")
env.cmd("ft.create", "idx", "schema", "t", "TEXT")
env.cmd("HSET", "doc1", "t", "d")
env.cmd("HSET", "doc2", "t", "dd")
env.cmd("HSET", "doc3", "t", "ddd")
env.cmd("HSET", "doc4", "t", "dde")
env.expect(debug_cmd(), "dump_terms", "idx").equal(['d', 'dd', 'ddd', 'dde'])
# The terms trie structure as this point looks like this: X -d> X -d> X -d> X, -e> X
# That is, there root node with a single child which is "d", which has another single child which is "d",
# that have two children which are "d" and "e".
# When we remove "d" from the try, we optimize and merge children that have a single child. Bug was in that
# merge operation that didn't copy properly the children keys array ("d" and "e" in our case), so that we only
# copied half of the children to the new merged node. Then, when deleting "dde", we couldn't find it in trie, and
# was left undeleted (inflating the memory as a consequence).
env.cmd("DEL", "doc1")
env.cmd(debug_cmd(), "GC_FORCEINVOKE", "idx")
# Validate that we actually delete "dde" and that in doesn't exist in the trie.
env.cmd("DEL", "doc4")
env.cmd(debug_cmd(), "GC_FORCEINVOKE", "idx")
env.expect(debug_cmd(), "dump_terms", "idx").equal(['dd', 'ddd'])
@skip()
def test_mod_4374(env):
conn = getConnectionByEnv(env)
env.cmd('FT.CREATE', 'idx', 'SCHEMA', 't', 'TEXT')
for i in range(10):
conn.execute_command('HSET', i, 't', 'val')
conn.execute_command('HSET', 10, 't', 'unique')
# the score of doc 10 is 6 without coordinator, and it is 4 with coordinator (3 shards)
print(conn.execute_command('FT.SEARCH', 'idx', 'val|unique', 'withscores', 'nocontent'))
@skip()
def test_mod_4375(env):
conn = getConnectionByEnv(env)
env.cmd('FT.CREATE', 'idx', 'SCHEMA', 't', 'TEXT', 'n', 'NUMERIC')
for i in range(10):
if i%2==0:
conn.execute_command('HSET', i, 't', 'even', 'n', i)
else:
conn.execute_command('HSET', i, 't', 'odd', 'n', i)
# Expected results are: ['0', '2', '4', '1', '3', '5', '6', '8']
print(conn.execute_command('FT.SEARCH', 'idx', '(-@t:even | @n:[0 5])', 'nocontent', 'dialect', '2'))
# After setting this configuration, we're getting: ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9']
conn.execute_command(config_cmd(), 'set', 'union_iterator_heap', '1')
print(conn.execute_command('FT.SEARCH', 'idx', '(-@t:even | @n:[0 5])', 'nocontent', 'dialect', '2'))
@skip(cluster=False) # This test is only relevant for cluster
def test_mod_6557(env: Env):
# Set validation timeout to 1ms so that we won't wait for the invalid topology to be validated
env.expect(config_cmd(), 'SET', 'TOPOLOGY_VALIDATION_TIMEOUT', '1').ok()
# Set topology to an invalid one (assuming port 9 is not open)
env.expect('SEARCH.CLUSTERSET',
'MYID',
'1',
'RANGES',
'2',
'SHARD',
'1',
'SLOTRANGE',
'0',
'8191',
'ADDR',
'127.0.0.1:9',
'MASTER',
'SHARD',
'2',
'SLOTRANGE',
'8192',
'16383',
'ADDR',
'127.0.0.1:9',
'MASTER'
).ok()
# Verify that `FT.CREATE` queries are not hanging and return an error
env.expect('FT.CREATE', 'idx', 'SCHEMA', 't', 'TEXT').error().contains('Could not distribute command')
def test_mod6186(env):
env.expect('FT.CREATE idx SCHEMA txt1 TEXT').equal('OK')
env.expect('FT.EXPLAIN idx abc*').equal('PREFIX{abc*}\n')
env.expect('FT.EXPLAIN idx *abc').equal('SUFFIX{*abc}\n')
env.expect('FT.EXPLAIN idx *abc*').equal('INFIX{*abc*}\n')
if not env.isCluster():
# FT.EXPLAINCLI is not supported by the coordinator
env.expect('FT.EXPLAINCLI idx abc*').equal(['PREFIX{abc*}', ''])
env.expect('FT.EXPLAINCLI idx *abc').equal(['SUFFIX{*abc}', ''])
env.expect('FT.EXPLAINCLI idx *abc*').equal(['INFIX{*abc*}', ''])
@skip(cluster=True)
def test_mod6510_vecsim_hybrid_adhoc_timeout(env):
dim = 1000
n_vectors = 50000
env.expect(config_cmd(), 'set', 'ON_TIMEOUT', 'FAIL').ok()
# Create HNSW index which is large enough, so we'll get timeout later on.
env.expect(f'FT.CREATE idx SCHEMA v VECTOR HNSW 10 DIM {dim} DISTANCE_METRIC L2 TYPE FLOAT32 M 2 EF_CONSTRUCTION 5'
f' t text').equal('OK')
for i in range(n_vectors):
env.expect('HSET', i, 'v', create_np_array_typed(np.random.rand(dim)).tobytes(), 't', f'meta data').equal(2)
# There was a bug causing this to unlock the tiered HNSW index locks twice (in case of timeout + adhoc BF mode).
query_vec = create_np_array_typed(np.random.rand(dim))
env.expect('FT.SEARCH', 'idx', 'meta=>[KNN 5 @v $vec_param HYBRID_POLICY ADHOC_BF]', 'NOCONTENT',
'PARAMS', 2, 'vec_param', query_vec.tobytes(), 'TIMEOUT', 1, 'DIALECT', 2)\
.error().contains('SEARCH_TIMEOUT Timeout limit was reached')
# Then, when we delete inplace and tried to acquire the locks again for write, we got a deadlock.
env.expect('DEL 0').equal(1)
@skip(cluster=False)
def test_mod_6541(env: Env):
env.expect('FT.CREATE', 'idx', 'SCHEMA', 't', 'TEXT', 'n', 'NUMERIC').ok()
cmds = [
('FT.SEARCH', 'idx', '*'),
('FT.AGGREGATE', 'idx', '*'),
('FT.CURSOR', 'READ', 'idx', '0'),
('FT.PROFILE', 'idx', 'SEARCH', 'QUERY', '*'),
('FT.PROFILE', 'idx', 'AGGREGATE', 'QUERY', '*'),
('FT.INFO', 'idx'),
('FT.SPELLCHECK', 'idx', 'foo'),
('FT.ALIASADD', 'alias', 'idx'),
# Deprecated commands
('FT.TAGVALS', 'idx', 't'),
('FT.MGET', 'idx', 'doc1', 'doc2'),
]
def expect_error(cmd):
return f'Cannot perform `{cmd[0]}`: Cannot block'
# Test MULTI/EXEC
for cmd in cmds:
env.expect('MULTI').ok()
env.expect(*cmd).equal('QUEUED')
res = env.cmd('EXEC')
env.assertEqual(len(res), 1, message=cmd[0])
env.assertIsInstance(res[0], redis_exceptions.ResponseError)
env.assertEqual(str(res[0]), expect_error(cmd))
# Test Lua
for cmd in cmds:
env.expect('EVAL', f'return redis.call{cmd}', '0').error().contains(expect_error(cmd))
@skip(cluster=True)
def test_4732(env):
'''
Test tokenizing text with an escaped backslash followed by a delimiter
(no need to test on cluster since only parser is tested)
'''
env.expect('FT.CREATE idx SCHEMA txt TEXT').equal('OK')
env.cmd('hset', 'doc1', 'txt', 'hello\\\\,world')
env.cmd('hset', 'doc2', 'txt', 'hello\\\\ world')
env.cmd('hset', 'doc3', 'txt', 'hello,world')
env.cmd('hset', 'doc4', 'txt', 'hello world')
env.expect('FT.SEARCH', 'idx', '@txt:(hello\\\\)', 'NOCONTENT').equal([2, 'doc1', 'doc2'])
env.expect('FT.SEARCH', 'idx', '@txt:(world)', 'NOCONTENT').equal([4, 'doc1', 'doc2', 'doc3', 'doc4'])
def test_mod_7252(env: Env):
env.expect('FT.CREATE idx SCHEMA neg NUMERIC').ok()
with env.getClusterConnectionIfNeeded() as conn:
[conn.execute_command('HSET', i, 'neg', -i) for i in range(1, 11)]
# Find the maximum negative value. The expected result is -1 (from [-10..-1])
env.expect('FT.AGGREGATE', 'idx', '*', 'GROUPBY', '0', 'REDUCE', 'MAX', '1', '@neg', 'AS', 'max').equal([1, ['max', '-1']])
def test_unsafe_simpleString_values():
env = Env(protocol=3) # Some cases only occur in RESP3
unsafe_index = 'unsafe\r\nindex'
unsafe_field = 'unsafe\r\nfield'
unsafe_value = 'unsafe\r\nvalue'
escape = lambda s: s.replace('\r', '\\r').replace('\n', '\\n')
# Test creating an index with unsafe name
env.expect('FT.CREATE', unsafe_index, 'PREFIX', '1', unsafe_value, 'SCHEMA', 't', 'TEXT').ok()
# Normalize output type across RESP2/RESP3 (server may return a list or set).
env.expect('FT._LIST').apply(lambda x: set(x)).equal({escape(unsafe_index)})
info = index_info(env, unsafe_index)
env.assertEqual(info['index_name'], escape(unsafe_index))
env.assertEqual(info['index_definition']['prefixes'], [escape(unsafe_value)])
# Test creating a field with unsafe name (and a tag field with unsafe separator)
env.expect('FT.ALTER', unsafe_index, 'SCHEMA', 'ADD', unsafe_field, 'TAG', 'SEPARATOR', '\n').ok()
tag_info = index_info(env, unsafe_index)['attributes'][-1]
expected = {'identifier': escape(unsafe_field), 'attribute': escape(unsafe_field), 'SEPARATOR': '\\n'}
[env.assertEqual(tag_info[k], v, message=k) for k, v in expected.items()]
# Test indexing failure report
env.expect('FT.ALTER', unsafe_index, 'SCHEMA', 'ADD', 'numval', 'NUMERIC').ok()
with env.getClusterConnectionIfNeeded() as conn:
conn.execute_command('HSET', unsafe_value, 'numval', unsafe_value)
error_info = index_info(env, unsafe_index)['Index Errors']
env.assertEqual(error_info['indexing failures'], 1)
env.assertContains(escape(unsafe_value), error_info['last indexing error'])
env.assertEqual(error_info['last indexing error key'], unsafe_value) # key is not escaped
# Test search with unsafe value
with env.getClusterConnectionIfNeeded() as conn:
conn.execute_command('HSET', unsafe_value, 't', 'hello', 'numval', 0, unsafe_field, 'tag\r1\ntag\r2')
get_results = lambda resp3_reply: resp3_reply['results']
expected = [{'id': unsafe_value, 'extra_attributes': {unsafe_field: 'tag\r1\ntag\r2'}, 'values': []}]
env.expect('FT.SEARCH', unsafe_index, '*', 'SORTBY', unsafe_field, 'RETURN', 1, unsafe_field).apply(get_results).equal(expected)
expected = [{'id': unsafe_value, 'values': []}]
env.expect('FT.SEARCH', unsafe_index, '*', 'SORTBY', unsafe_field, 'RETURN', 0).apply(get_results).equal(expected)
def test_mod_7463(env: Env):
env.expect('FT.CREATE', 'idx', 'SCHEMA', 'name', 'TEXT').ok()
with env.getClusterConnectionIfNeeded() as conn:
conn.execute_command('HSET', 'doc1', 'name', 'hello kitty')
env.expect('FT.SEARCH', 'idx', 'kitti').equal([1, 'doc1', ['name', 'hello kitty']])
env.expect('FT.SEARCH', 'idx', 'kitti', 'VERBATIM').equal([0])
env.expect('FT.AGGREGATE', 'idx', 'kitti', 'LOAD', '*').equal([1, ['name', 'hello kitty']])
env.expect('FT.AGGREGATE', 'idx', 'kitti', 'VERBATIM', 'LOAD', '*').equal([0])
@skip(cluster=True)
def test_mod_7495(env: Env):
env.expect('FT.CREATE', 'idx', 'SCHEMA', 't', 'TEXT').ok()
# testing union of stopwords (at least the first 2 were required to reproduce the crash)
env.expect('FT.SEARCH', 'idx', '(is|the|a|of|in|and)', 'DIALECT', '2').equal([0]).noError()
env.cmd('HSET', 'doc1', 't', 'hello world')
expected = [1, 'doc1', ['t', 'hello world']]
# First non-stopword is found
env.expect('FT.SEARCH', 'idx', '(is|the|a|of|in|world)', 'DIALECT', '2').equal(expected).noError()
# First non-stopword is not found
env.expect('FT.SEARCH', 'idx', '(is|the|a|of|in|foo)', 'DIALECT', '2').equal([0]).noError()
env.expect('FT.SEARCH', 'idx', '(is|the|a|of|in|foo|world)', 'DIALECT', '2').equal(expected).noError()
@skip(cluster=True)
def test_mod_8142(env:Env):
env.expect('FT.CREATE', 'idx', 'SCHEMA', 't', 'TEXT').ok()
env.expect('FT.CREATE', 'idxOptimized', 'INDEXALL', 'ENABLE', 'SCHEMA', 't', 'TEXT').ok()
env.cmd('HSET', 'doc1', 't', 'city')
env.cmd('HSET', 'doc2', 't', 'cities')
score_opt = ['WITHSCORES', 'SCORER', 'TFIDF']
# Test with a term search
env.expect('FT.SEARCH', 'idx', 'city', *score_opt).equal([2, 'doc1', '3', ['t', 'city'], 'doc2', '1', ['t', 'cities']])
env.expect('FT.SEARCH', 'idx', 'cities', *score_opt).equal([2, 'doc2', '3', ['t', 'cities'], 'doc1', '1', ['t', 'city']])
# Test with an exact term search
env.expect('FT.SEARCH', 'idx', '"city"', *score_opt).equal([1, 'doc1', '2', ['t', 'city']])
env.expect('FT.SEARCH', 'idx', '"cities"', *score_opt).equal([1, 'doc2', '2', ['t', 'cities']])
# Test with an optional term search
env.expect('FT.SEARCH', 'idx', '~city', *score_opt).equal([2, 'doc1', '3', ['t', 'city'], 'doc2', '1', ['t', 'cities']])
env.expect('FT.SEARCH', 'idxOptimized', '~city', *score_opt).equal([2, 'doc1', '3', ['t', 'city'], 'doc2', '1', ['t', 'cities']])
env.expect('FT.SEARCH', 'idx', '~cities', *score_opt).equal([2, 'doc2', '3', ['t', 'cities'], 'doc1', '1', ['t', 'city']])
env.expect('FT.SEARCH', 'idxOptimized', '~cities', *score_opt).equal([2, 'doc2', '3', ['t', 'cities'], 'doc1', '1', ['t', 'city']])
# Test with an optional exact term search
env.expect('FT.SEARCH', 'idx', '~"city"', *score_opt).equal([2, 'doc1', '2', ['t', 'city'], 'doc2', '0', ['t', 'cities']])
env.expect('FT.SEARCH', 'idxOptimized', '~"city"', *score_opt).equal([2, 'doc1', '2', ['t', 'city'], 'doc2', '0', ['t', 'cities']])
env.expect('FT.SEARCH', 'idx', '~"cities"', *score_opt).equal([2, 'doc2', '2', ['t', 'cities'], 'doc1', '0', ['t', 'city']])
env.expect('FT.SEARCH', 'idxOptimized', '~"cities"', *score_opt).equal([2, 'doc2', '2', ['t', 'cities'], 'doc1', '0', ['t', 'city']])
# Test without a term search
env.expect('FT.SEARCH', 'idx', '-city', *score_opt).equal([0])
env.expect('FT.SEARCH', 'idxOptimized', '-city', *score_opt).equal([0])
env.expect('FT.SEARCH', 'idx', '-cities', *score_opt).equal([0])
env.expect('FT.SEARCH', 'idxOptimized', '-cities', *score_opt).equal([0])
# Test without an exact term search
env.expect('FT.SEARCH', 'idx', '-"city"', *score_opt).equal([1, 'doc2', '0', ['t', 'cities']])
env.expect('FT.SEARCH', 'idxOptimized', '-"city"', *score_opt).equal([1, 'doc2', '0', ['t', 'cities']])
env.expect('FT.SEARCH', 'idx', '-"cities"', *score_opt).equal([1, 'doc1', '0', ['t', 'city']])
env.expect('FT.SEARCH', 'idxOptimized', '-"cities"', *score_opt).equal([1, 'doc1', '0', ['t', 'city']])
# Test with an optional negated term search
env.expect('FT.SEARCH', 'idx', '~-city', *score_opt).equal([2, 'doc1', '0', ['t', 'city'], 'doc2', '0', ['t', 'cities']])
env.expect('FT.SEARCH', 'idxOptimized', '~-city', *score_opt).equal([2, 'doc1', '0', ['t', 'city'], 'doc2', '0', ['t', 'cities']])
env.expect('FT.SEARCH', 'idx', '~-cities', *score_opt).equal([2, 'doc1', '0', ['t', 'city'], 'doc2', '0', ['t', 'cities']])
env.expect('FT.SEARCH', 'idxOptimized', '~-cities', *score_opt).equal([2, 'doc1', '0', ['t', 'city'], 'doc2', '0', ['t', 'cities']])
# Test with an optional negated exact term search
env.expect('FT.SEARCH', 'idx', '~-"city"', *score_opt).equal([2, 'doc1', '0', ['t', 'city'], 'doc2', '0', ['t', 'cities']])
env.expect('FT.SEARCH', 'idxOptimized', '~-"city"', *score_opt).equal([2, 'doc1', '0', ['t', 'city'], 'doc2', '0', ['t', 'cities']])
env.expect('FT.SEARCH', 'idx', '~-"cities"', *score_opt).equal([2, 'doc1', '0', ['t', 'city'], 'doc2', '0', ['t', 'cities']])
env.expect('FT.SEARCH', 'idxOptimized', '~-"cities"', *score_opt).equal([2, 'doc1', '0', ['t', 'city'], 'doc2', '0', ['t', 'cities']])
# Verify that the vector search doesn't affect the scoring or result set
env.expect('FT.ALTER', 'idx', 'SKIPINITIALSCAN', 'SCHEMA', 'ADD', 'v', 'VECTOR', 'FLAT', '6', 'TYPE', 'FLOAT32', 'DIM', '2', 'DISTANCE_METRIC', 'L2').ok()
env.cmd('HSET', 'doc1', 'v', np.array([1, 1], dtype=np.float32).tobytes())
env.cmd('HSET', 'doc2', 'v', np.array([1, 2], dtype=np.float32).tobytes())
res1 = env.cmd('FT.SEARCH', 'idx', 'city', 'WITHSCORES', 'RETURN', '1', 't')
res2 = env.cmd('FT.SEARCH', 'idx', 'city=>[KNN 10 @v $BLOB]', 'WITHSCORES', 'RETURN', '1', 't', 'DIALECT', '2',
'PARAMS', 2, 'BLOB', np.array([1, 0], dtype=np.float32).tobytes())
env.assertEqual(res1, res2)
@skip(cluster=True)
def test_mod_7882(env:Env):
"""
We currently don't support searching for strings that are longer than 1024 characters in the Trie.
"""
env.expect('FT.CREATE', 'idx', 'SCHEMA', 't', 'TEXT').ok()
long_text = 'a'*1025
# All the queries below should return an error without crashing.
env.expect('FT.SEARCH', 'idx', '*' + long_text).error().contains('SUFFIX query string is too long')
env.expect('FT.SEARCH', 'idx', long_text + '*').error().contains('PREFIX query string is too long')
env.expect('FT.SEARCH', 'idx', '*' + long_text + '*').error().contains('INFIX query string is too long')
env.expect('FT.SEARCH', 'idx', "w'" + long_text + "'", 'DIALECT', '2').error().contains('Wildcard query string is too long')
@skip(cluster=True)
def test_mod_6783(env:Env):
n_max_sortable = 1024
n_docs = 10
step = 71 # Testing every possible number of sortables is too slow
# Add documents with a unique values for each sortable field, in unique random orders
orders = random.choices(list(itertools.permutations(range(n_docs))), k=n_max_sortable)
for field_id, vals in enumerate(orders):
for doc_id, val in enumerate(vals):
env.cmd('HSET', f'doc{doc_id}', f'f{field_id}', val)
expected = [sorted(range(n_docs), key=lambda x: order[x]) for order in orders]
expected = [[n_docs] + [f'doc{doc_id}' for doc_id in exp] for exp in expected]
for n_sortables in range(1, n_max_sortable + 1, step):
env.expect('FT._DROPINDEXIFX', 'idx').ok()
schema = sum([['f'+str(i), 'NUMERIC', 'SORTABLE'] for i in range(n_sortables)], [])
env.expect('FT.CREATE', 'idx', 'SCHEMA', *schema).ok()
waitForIndex(env)
for i in range(n_sortables):
res = env.cmd('FT.SEARCH', 'idx', '*', 'SORTBY', f'f{i}', 'NOCONTENT')
env.assertEqual(res, expected[i], message=f'Failed on field f{i} with {n_sortables} sortables')
@skip(cluster=True)
def test_mod_8589(env:Env):
env.expect('FT.CREATE', 'idx', 'SCHEMA', 't', 'TEXT', 'v', 'VECTOR', 'FLAT', '6', 'TYPE', 'FLOAT32', 'DIM', '2', 'DISTANCE_METRIC', 'L2').ok()
env.cmd('HSET', 'doc1', 'v', '????????', 't', 'foo bar foo') # Max frequency is 2 (foo)
docinfo = to_dict(env.cmd(debug_cmd(), 'DOCINFO', 'idx', 'doc1', 'REVEAL'))
env.assertEqual(docinfo['max_freq'], 2)
@skip(cluster=True)
def test_mod_8568(env:Env):
env.expect('FT.CREATE', 'idx', 'SCHEMA', 'g', 'GEO').ok()
env.expect('HSET', 'doc1', 'g', '1.1,1.1').equal(1)
env.expect('HSET', 'doc2', 'g', '1.2,1.2').equal(1)
expected = [1, 'doc1', ['g', '1.1,1.1']]
env.expect('FT.SEARCH', 'idx', '*', 'GEOFILTER', 'g', '1.1', '1.1', '1', 'km').equal(expected)
env.expect('FT.SEARCH', 'idx', '*', 'GEOFILTER', 'g', '1.1', '1.1', '1', 'km',
'GEOFILTER', 'g', '1.1', '1.1', '1000', 'km').equal(expected)
@skip(cluster=True)
def test_mod_6786(env:Env):
# Test search of long term (>128) inside text field
MAX_NORMALIZE_SIZE = 128
env.expect('FT.CREATE', 'idx', 'SCHEMA', 't', 'TEXT').ok()
long_term = 'A'*(MAX_NORMALIZE_SIZE+1)
text_with_long_term = ' '.join([long_term, long_term[:MAX_NORMALIZE_SIZE//2]])
env.cmd('HSET', 'doc1', 't', text_with_long_term)
# Searching for the long term should return the document
# Before fix, the long term was partially normalized and the document was not found
env.expect('FT.SEARCH', 'idx', long_term).equal([1, 'doc1', ['t', text_with_long_term]])
@skip(cluster=False)
def test_mod_7609(env:Env):
# Create the same named index on all shards, but with different schemas
for i in range(1, env.shardsCount + 1):
con = env.getConnection(i)
con.execute_command('DEBUG', 'MARK-INTERNAL-CLIENT') # required for running the internal `_FT.CREATE` command
schema = []
for j in range(i):
schema.extend(['f'+str(j), 'TEXT'])
con.execute_command('_FT.CREATE', 'idx', 'SCHEMA', *schema)
env.expect('FT.INFO', 'idx').error().contains('Inconsistent index state')
@skip(cluster=True)
def test_mod_8561(env:Env):
env.expect(config_cmd(), 'SET', 'FORK_GC_CLEAN_THRESHOLD', '0').ok()
env.expect('FT.CREATE', 'idx1', 'SCHEMA', 't', 'TEXT').ok()
env.expect('FT.CREATE', 'idx2', 'SCHEMA', 't', 'TAG').ok()
# Add a document with the term foo
env.cmd('HSET', '1', 't', 'foo')
# Add two documents with the terms foo and bar
env.cmd('HSET', '2', 't', 'foo,bar')
env.cmd('HSET', '3', 't', 'foo,bar')
# Delete the last document with the term foo
env.cmd('DEL', '3')
# Run GC to remove the deleted document
forceInvokeGC(env, 'idx1')
forceInvokeGC(env, 'idx2')
# Search
expected = [1, '2', ['t', 'foo,bar']]
env.expect('FT.SEARCH', 'idx1', 'bar foo').noError().equal(expected)
env.expect('FT.SEARCH', 'idx2', "@t:{bar} @t:{foo}").noError().equal(expected)
import time
@skip(cluster=True)
def test_mod_8695():
env = Env(moduleArgs='DEFAULT_DIALECT 2')
env.expect('FT.CREATE', 'idx', 'SCHEMA', 't', 'TEXT',
'v', 'VECTOR', 'FLAT', '6', 'TYPE', 'FLOAT32', 'DIM', '2', 'DISTANCE_METRIC', 'L2').ok()
env.cmd('HSET', 'doc1', 't', 'foo', 'v', '????????')
env.cmd('HSET', 'doc2', 't', 'bar', 'v', '????????')
env.cmd('HSET', 'doc3', 't', 'foo bar', 'v', '????????')
# Test highlighting
res1 = env.cmd('FT.SEARCH', 'idx', 'foo',
'HIGHLIGHT', 'FIELDS', 1, 't', 'RETURN', 1, 't')
res2 = env.cmd('FT.SEARCH', 'idx', 'foo=>[KNN 10 @v $BLOB]', 'PARAMS', 2, 'BLOB', '????????',
'HIGHLIGHT', 'FIELDS', 1, 't', 'RETURN', 1, 't')
env.assertEqual(res1, res2)
res1 = env.cmd('FT.SEARCH', 'idx', 'foo|bar',
'HIGHLIGHT', 'FIELDS', 1, 't', 'RETURN', 1, 't')
res2 = env.cmd('FT.SEARCH', 'idx', '(foo|bar)=>[KNN 10 @v $BLOB]', 'PARAMS', 2, 'BLOB', '????????',
'HIGHLIGHT', 'FIELDS', 1, 't', 'RETURN', 1, 't')
env.assertEqual(res1, res2)
res1 = env.cmd('FT.SEARCH', 'idx', 'foo bar',
'HIGHLIGHT', 'FIELDS', 1, 't', 'RETURN', 1, 't')
res2 = env.cmd('FT.SEARCH', 'idx', '(foo bar)=>[KNN 10 @v $BLOB]', 'PARAMS', 2, 'BLOB', '????????',
'HIGHLIGHT', 'FIELDS', 1, 't', 'RETURN', 1, 't')
env.assertEqual(res1, res2)
# Test vector with highlight only (implicit return)
env.expect('FT.SEARCH', 'idx', 'foo=>[KNN 10 @v $BLOB as score]', 'PARAMS', 2, 'BLOB', '????????',
'HIGHLIGHT', 'FIELDS', 1, 't', ).noError().equal(
[2, 'doc1', ['score', '0', 't', 'foo', 'v', '????????'], 'doc3', ['score', '0', 't', 'foo bar', 'v', '????????']])
# Test vector with highlight and explicit return
env.expect('FT.SEARCH', 'idx', 'foo=>[KNN 10 @v $BLOB as score]', 'PARAMS', 2, 'BLOB', '????????',
'RETURN', 2, 't', 'score', 'HIGHLIGHT', 'FIELDS', 1, 't').noError().equal(
[2, 'doc1', ['score', '0', 't', 'foo'], 'doc3', ['score', '0', 't', 'foo bar']])
# Test that we get the same results (with scores) regardless of the order of the arguments
res1 = env.cmd('FT.SEARCH', 'idx', 'foo=>[KNN 10 @v $BLOB as score]', 'PARAMS', 2, 'BLOB', '????????',
'SORTBY', 'score', 'WITHSCORES')
res2 = env.cmd('FT.SEARCH', 'idx', 'foo=>[KNN 10 @v $BLOB as score]', 'PARAMS', 2, 'BLOB', '????????',
'WITHSCORES', 'SORTBY', 'score')
env.assertEqual(res1, res2)
# Test vector with AGGREGATE and scores
env.expect('FT.AGGREGATE', 'idx', 'foo=>[KNN 10 @v $BLOB as score]', 'PARAMS', 2, 'BLOB', '????????', 'ADDSCORES', 'SCORER', 'TFIDF', 'TIMEOUT', 0).noError().apply(lambda res: res[1:]).equal(
[['score', '0', '__score', '1'], ['score', '0', '__score', '1']])
@skip(cluster=True)
def test_mod_9423(env:Env):
env.expect('FT.CREATE', 'idx', 'SCHEMA', 't', 'TEXT').ok()
env.cmd('HSET', 'doc1', 'n', '1') # Document with no text
# Expect the document score to be 0, since it has no text
expected = [1, 'doc1', '0', ['n', '1']]
env.expect('FT.SEARCH', 'idx', '*', 'WITHSCORES', 'SCORER', 'TFIDF').equal(expected)
env.expect('FT.SEARCH', 'idx', '*', 'WITHSCORES', 'SCORER', 'TFIDF.DOCNORM').equal(expected)
expected = [1, 'doc1', ['0', 'Document max frequency is 0'], ['n', '1']]
env.expect('FT.SEARCH', 'idx', '*', 'WITHSCORES', 'SCORER', 'TFIDF', 'EXPLAINSCORE').equal(expected)
expected = [1, 'doc1', ['0', 'Document length is 0'], ['n', '1']]
env.expect('FT.SEARCH', 'idx', '*', 'WITHSCORES', 'SCORER', 'TFIDF.DOCNORM', 'EXPLAINSCORE').equal(expected)
# Test that RedisModule_Yield is called while indexing in order to prevent master from killing the replica [MOD-8809]
@skip(cluster=True)
def test_mod_8809_single_index_single_field(env:Env):
# Configure yield every 10 operations
yield_every_n_ops = 10
env.expect(config_cmd(), 'SET', 'INDEXER_YIELD_EVERY_OPS', f'{yield_every_n_ops}').ok()
env.expect(config_cmd(), 'GET', 'INDEXER_YIELD_EVERY_OPS').equal([['INDEXER_YIELD_EVERY_OPS', f'{yield_every_n_ops}']])
# Reset yield counter
env.expect(debug_cmd(), 'YIELDS_COUNTER', 'RESET').ok()
initial_count = env.cmd(debug_cmd(), 'YIELDS_COUNTER', 'LOAD')
env.assertEqual(initial_count, 0, message="Initial yield counter should be 0")
# Create index
dimension = 128
env.cmd('FT.CREATE', 'idx', 'SCHEMA', 'v', 'VECTOR', 'HNSW', '6', 'TYPE', 'FLOAT32', 'DIM', dimension, 'DISTANCE_METRIC', 'L2')
# Add enough documents to trigger yields
num_docs = 1000
for i in range(num_docs):
vector = np.random.rand(1, dimension).astype(np.float32)
env.execute_command('HSET', f'doc{i}', 'v', vector.tobytes())
waitForIndex(env, 'idx')
# Check that yield was not called
yields_count = env.cmd(debug_cmd(), 'YIELDS_COUNTER', 'LOAD')
env.assertEqual(yields_count, 0, message="Yield should not have been called")
# Reload and check
env.broadcast('SAVE')
env.broadcast('DEBUG RELOAD NOSAVE')
waitForIndex(env, 'idx')
env.expect(config_cmd(), 'GET', 'INDEXER_YIELD_EVERY_OPS').equal([['INDEXER_YIELD_EVERY_OPS', f'{yield_every_n_ops}']])
# Verify the number of yields
expected_min_yields = num_docs // yield_every_n_ops
yields_count = env.cmd(debug_cmd(), 'YIELDS_COUNTER', 'LOAD')
env.assertGreaterEqual(yields_count, expected_min_yields,
message=f"Expected at least {expected_min_yields} yields, got {yields_count}")
# Test with different configuration
yields_every_n_ops = 5
env.expect(config_cmd(), 'SET', 'INDEXER_YIELD_EVERY_OPS', f'{yield_every_n_ops}').ok()
env.expect(debug_cmd(), 'YIELDS_COUNTER', 'RESET').ok()
# Reload and check
env.broadcast('SAVE')
env.broadcast('DEBUG RELOAD NOSAVE')
waitForIndex(env, 'idx')
yields_count = env.cmd(debug_cmd(), 'YIELDS_COUNTER', 'LOAD')
expected_min_yields = num_docs // yield_every_n_ops
env.assertGreaterEqual(yields_count, expected_min_yields,
message=f"Expected at least {expected_min_yields} yields, got {yields_count}")
@skip(cluster=True)
def test_mod_8809_multi_index_multi_fields(env:Env):
# Configure yield every 10 operations
yield_every_n_ops = 10
env.expect(config_cmd(), 'SET', 'INDEXER_YIELD_EVERY_OPS', f'{yield_every_n_ops}').ok()
env.expect(config_cmd(), 'GET', 'INDEXER_YIELD_EVERY_OPS').equal([['INDEXER_YIELD_EVERY_OPS', f'{yield_every_n_ops}']])
# Reset yield counter
env.expect(debug_cmd(), 'YIELDS_COUNTER', 'RESET').ok()
initial_count = env.cmd(debug_cmd(), 'YIELDS_COUNTER', 'LOAD')
env.assertEqual(initial_count, 0, message="Initial yield counter should be 0")
# Create index
dimension = 128
env.cmd('FT.CREATE', 'idx', 'SCHEMA', 'num', 'NUMERIC', 'v', 'VECTOR', 'HNSW', '6', 'TYPE', 'FLOAT32', 'DIM', dimension, 'DISTANCE_METRIC', 'L2')
env.cmd('FT.CREATE', 'idx2', 'SCHEMA', 't', 'TEXT', 'v', 'VECTOR', 'HNSW', '6', 'TYPE', 'FLOAT32', 'DIM', dimension, 'DISTANCE_METRIC', 'L2')
env.cmd('FT.CREATE', 'idx3', 'SCHEMA', 'geom', 'GEOSHAPE', 'tag', 'TAG' ,'t2', 'TEXT')
# Add enough documents to trigger yields
num_docs = 1000
for i in range(num_docs):
vector = np.random.rand(1, dimension).astype(np.float32)
env.execute_command('HSET', f'doc{i}', 'v', vector.tobytes(), 'num', i, 't', f'text {i}', 't2', f'text2 {i}', 'geom', f'POINT({i%10} {i%15})', 'tag', f'tag{i%10}')
waitForIndex(env, 'idx')
waitForIndex(env, 'idx2')
waitForIndex(env, 'idx3')
# Reload and check
env.broadcast('SAVE')
env.broadcast('DEBUG RELOAD NOSAVE')
waitForIndex(env, 'idx')
waitForIndex(env, 'idx2')
waitForIndex(env, 'idx3')
# Check that yield was called
yields_count = env.cmd(debug_cmd(), 'YIELDS_COUNTER', 'LOAD')
env.assertGreater(yields_count, 0, message="Yield should have been called at least once")
# Verify the number of yields
expected_min_yields = 7 * num_docs // yield_every_n_ops
env.assertGreaterEqual(yields_count, expected_min_yields,
message=f"Expected at least {expected_min_yields} yields, got {yields_count}")
# Test with different configuration
yield_every_n_ops = 5
env.expect(config_cmd(), 'SET', 'INDEXER_YIELD_EVERY_OPS', f'{yield_every_n_ops}').ok()
env.expect(debug_cmd(), 'YIELDS_COUNTER', 'RESET').ok()
# Reload and check
env.broadcast('SAVE')
env.broadcast('DEBUG RELOAD NOSAVE')
waitForIndex(env, 'idx')
waitForIndex(env, 'idx2')
waitForIndex(env, 'idx3')
env.expect(config_cmd(), 'GET', 'INDEXER_YIELD_EVERY_OPS').equal([['INDEXER_YIELD_EVERY_OPS', f'{yield_every_n_ops}']])
yields_count = env.cmd(debug_cmd(), 'YIELDS_COUNTER', 'LOAD')
expected_min_yields = 7 * num_docs // yield_every_n_ops
env.assertGreaterEqual(yields_count, expected_min_yields,
message=f"Expected at least {expected_min_yields} yields, got {yields_count}")
def _mod_8157(env:Env):
"""
Test missing profile info on aggregate query
"""
shard_chunk_size = 1000 # Hardcoded chunk size from the shards
def verify_profile(reply):
if env.protocol == 2:
# RESP2 returns a list of lists
profile = reply[1]
else:
# RESP3 returns a dictionary
profile = reply['Profile']
profile = to_dict(profile)
env.assertContains('Shards', profile, message='missing `Shards` section', depth=1)
env.assertEqual(len(profile['Shards']), env.shardsCount, message='missing some shards profile info', depth=1)
# Case 1: Profile info arrives in an empty reply (some shards have no documents,
# one have exactly `shard_chunk_size` documents, so another read is required to get EOF)
env.expect('FT.CREATE', 'idx1', 'PREFIX', '1', 'case1:', 'SCHEMA', 't', 'TEXT').ok()
# Add to some shard exactly `shard_chunk_size` documents
with env.getClusterConnectionIfNeeded() as conn:
for i in range(shard_chunk_size):
# Use `{x}` suffix to ensure that the documents are added to the same shard
conn.execute_command('HSET', f'case1:{i}{{x}}', 't', 'foo')
reply = env.cmd('FT.PROFILE', 'idx1', 'AGGREGATE', 'QUERY', '*')
verify_profile(reply)
# Case 2: Profile info arrives but the coordinator doesn't consume the reply fully
# (previously, we didn't pass the reply to the profile reply aggregation)
env.expect('FT.CREATE', 'idx2', 'PREFIX', '1', 'case2:', 'SCHEMA', 't', 'TEXT').ok()
with env.getClusterConnectionIfNeeded() as conn:
# `chunk_size` documents spread across all shards, so each shard will have less than `shard_chunk_size` documents
for i in range(shard_chunk_size):
conn.execute_command('HSET', f'case2:{i}', 't', 'foo')
# Search for a bit less than `shard_chunk_size` documents, so the coordinator will not deplete the last shard reply
reply = env.cmd('FT.PROFILE', 'idx2', 'AGGREGATE', 'QUERY', '*', 'LIMIT', '0', str(int(shard_chunk_size * 0.95)))
verify_profile(reply)
@skip(cluster=False, min_shards=2)
def test_mod_8157_RESP2():
_mod_8157(Env(protocol=2))
@skip(cluster=False, min_shards=2)
def test_mod_8157_RESP3():
_mod_8157(Env(protocol=3))
@skip(cluster=True)
def test_mod_11975(env: Env):
env.expect('FT.CREATE', 'idx', 'SCHEMA', 't', 'TEXT').ok()
env.expect('FT.SEARCH', 'idx', '@t:("*")', 'DIALECT', '2').equal([0])
def check_threads(env, expected_num_threads_alive, expected_n_threads):
env.assertEqual(getWorkersThpoolStats(env)['numThreadsAlive'], expected_num_threads_alive, depth=1, message='numThreadsAlive should match num_threads_alive')
env.assertEqual(getWorkersThpoolNumThreads(env), expected_n_threads, depth=1, message='n_threads should match WORKERS')
def test_mod_11658_avoid_deadlock_while_reducing_num_workers():
"""
Test that changing WORKERS from high to 0 under load doesn't cause unresponsiveness.
This test:
1. Starts with WORKERS=8 (simulating QPF=8)
2. Creates an index and loads data
3. Runs concurrent queries in background threads (100 threads, no delays)
4. Changes WORKERS to 0 (simulating QPF=0 change)
5. Verifies the shard remains responsive
"""
# This test requires coordinator mode (OSS cluster)
# Start with 8 workers (simulating QPF=8)
env = Env(moduleArgs='WORKERS 8', enableDebugCommand=True)
# Create index with multiple field types to make queries more complex
env.expect('FT.CREATE', 'idx', 'SCHEMA',
'title', 'TEXT', 'WEIGHT', '2.0',
'body', 'TEXT',
'price', 'NUMERIC', 'SORTABLE',
'category', 'TAG',
'location', 'GEO').ok()
# Load a significant amount of data to make queries take time
conn = getConnectionByEnv(env)
n_docs = 1000
categories = ['electronics', 'books', 'clothing', 'food', 'toys']
for i in range(n_docs):
conn.execute_command('HSET', f'doc{i}',
'title', f'Product {i} title with searchable text',
'body', f'This is the body of document {i} with more searchable content',
'price', random.randint(10, 1000),
'category', random.choice(categories),
'location', f'{random.uniform(-90, 90)},{random.uniform(-180, 180)}')
waitForIndex(env, 'idx')
# Verify initial state
initial_workers = env.cmd(config_cmd(), 'GET', 'WORKERS')
env.assertEqual(initial_workers, [['WORKERS', '8']])
# Flag to control query threads
stop_queries = threading.Event()
query_success_count = [0] # Use list to allow modification in thread
def run_queries():
"""Run various queries continuously until stopped"""
local_conn = env.getConnection()
while not stop_queries.is_set():
# Mix of different query types
queries = [
['FT.SEARCH', 'idx', '*', 'LIMIT', '0', '10'],
['FT.SEARCH', 'idx', 'searchable', 'LIMIT', '0', '10'],
['FT.SEARCH', 'idx', '@category:{electronics}', 'LIMIT', '0', '10'],
['FT.SEARCH', 'idx', '*', 'SORTBY', 'price', 'ASC', 'LIMIT', '0', '10'],
['FT.AGGREGATE', 'idx', '*', 'GROUPBY', '1', '@category', 'REDUCE', 'COUNT', '0', 'AS', 'count'],
]
query = random.choice(queries)
_ = local_conn.execute_command(*query)
query_success_count[0] += 1
# Start multiple query threads to simulate concurrent load
# Increase thread count to maximize likelihood of hitting the race condition
# The bug requires worker threads to be actively processing queries when
# the config change happens
num_query_threads = 20
query_threads = []
for i in range(num_query_threads):
t = threading.Thread(target=run_queries, name=f'QueryThread-{i}')
t.start()
query_threads.append(t)
# Let queries run for a bit to establish load
# Increase time to ensure all threads are actively querying
pre_count = 0
with TimeLimit(10):
while (pre_count < 100):
time.sleep(0.1)
pre_count = query_success_count[0]
# Verify queries are running successfully
initial_success = query_success_count[0]
env.assertTrue(initial_success > 0, message="Queries should be running successfully before config change")
# I can check the thread pool state after the thpool is initialized by the first query
check_threads(env, 8, 8)
# Now change WORKERS to 0 (simulating QPF change from 8 to 0)
# This is the critical moment that triggers the bug
env.debugPrint("Changing WORKERS from 8 to 0 while queries are running...", force=True)
pre_count = query_success_count[0]
env.debugPrint(f"Query success count before config change: {pre_count}", force=True)
# The bug: This command may hang indefinitely if worker threads are blocked
# waiting for coordinator connections that were stopped by MRConnManager_Shrink
env.expect(config_cmd(), 'SET', 'WORKERS', '0').ok()
# Verify the config change is reflected in the getter
new_workers = env.cmd(config_cmd(), 'GET', 'WORKERS')
env.assertEqual(new_workers, [['WORKERS', '0']])
post_count = pre_count
with TimeLimit(10):
while (post_count == pre_count):
time.sleep(0.1)
post_count = query_success_count[0]
post_count = query_success_count[0]
env.debugPrint(f"Query success count after config change: {post_count}", force=True)
env.assertGreater(post_count, pre_count, message="Queries should continue running after config change")
with TimeLimit(10):
while (getWorkersThpoolStats(env)['numThreadsAlive'] != 0 or getWorkersThpoolNumThreads(env) != 0):
time.sleep(0.1)
check_threads(env, 0, 0)
# Verify the config change took effect
# Critical test: Verify Redis is still responsive
# This is where the bug manifests - Redis becomes unresponsive
try:
# Try to PING - this should work even with WORKERS=0
ping_result = env.cmd('PING')
env.assertContains(ping_result, ['PONG', True], message="Redis should respond to PING after WORKERS change")
# Try a simple query - this should work on the main thread
search_result = env.cmd('FT.SEARCH', 'idx', '*', 'LIMIT', '0', '1')
env.assertTrue(search_result is not None, message="Search should work after WORKERS change")
# Try to add a new document
add_result = conn.execute_command('HSET', 'newdoc', 'title', 'test', 'price', '100', 'category', 'test')
env.assertContains(add_result, [3, True], message="Should be able to add documents after WORKERS change")
except Exception as e:
env.debugPrint(f"CRITICAL: Redis became unresponsive after WORKERS change: {e}", force=True)
raise AssertionError(f"Redis became unresponsive after WORKERS change (MOD-11658 reproduced): {e}")
# Stop query threads
stop_queries.set()
for t in query_threads:
t.join(timeout=5)
# Final verification: Redis should still be fully functional
final_ping = env.cmd('PING')
env.assertTrue(final_ping in ['PONG', True], message="Redis should still respond to PING at end of test")
final_search = env.cmd('FT.SEARCH', 'idx', '*', 'LIMIT', '0', '5')
env.assertTrue(final_search[0] > 0, message="Search should return results at end of test")
check_threads(env, 0, 0)
@skip(cluster=False)
def test_mod_12493(env:Env):
env.expect('FT.CREATE', 'idx', 'SCHEMA', 'n', 'NUMERIC').ok()
# Add enough documents so 4 reads are needed from each shard to read all results (chunk size is 1000)
n_docs = 3200 * env.shardsCount
with env.getClusterConnectionIfNeeded() as conn:
for i in range(n_docs):
conn.execute_command('HSET', f'doc{i}', 'n', i)
# Create a cursor
_, cursor = env.cmd('FT.AGGREGATE', 'idx', '*', 'WITHCURSOR')
# Check that there are pending cursors on all shards
env.assertEqual(to_dict(index_info(env)['cursor_stats'])['index_total'], env.shardsCount)
# Read another env.shardCount - 1 times. Expecting that each read will read 1000 results from each shard,
# so after env.shardsCount reads (including the initial aggregate), we will trigger another read from each shard
for _ in range(env.shardsCount - 1):
_, cursor = env.cmd('FT.CURSOR', 'READ', 'idx', cursor)
# Check again that there are pending cursors on all shards
env.assertEqual(to_dict(index_info(env)['cursor_stats'])['index_total'], env.shardsCount)
# Check command stats for internal cursors command. We expect a single one (READ) on each shard
for i, con in enumerate(env.getOSSMasterNodesConnectionList()):
stats = con.execute_command('INFO', 'COMMANDSTATS')['cmdstat__FT.CURSOR|READ']
env.assertEqual(stats['calls'], 1, message=f'Expected 1 call on shard {i}, got {stats["calls"]}')
# Delete the cursor. This should delete the internal cursors on all shards.
# If we call READ instead, they won't be deleted or depleted (3rd read, and we have 4 chunks), and the test will fail.
env.expect('FT.CURSOR', 'DEL', 'idx', cursor).ok()
# Check that the internal cursors were deleted on all shards. This happens asynchronously
with TimeLimit(10, 'Internal cursors were not deleted within the time limit'):
while to_dict(index_info(env)['cursor_stats'])['index_total'] != 0:
time.sleep(0.1)
# Expect another call on each shard for the DEL command
for i, con in enumerate(env.getOSSMasterNodesConnectionList()):
stats = con.execute_command('INFO', 'COMMANDSTATS')['cmdstat__FT.CURSOR|DEL']
env.assertEqual(stats['calls'], 1, message=f'Expected 1 call on shard {i}, got {stats["calls"]}')
def test_mod_13010(env):
"""Test coherence between aggregate queries with and without groupby"""
conn = getConnectionByEnv(env)
# Create index with schema matching the query requirements
env.expect(
'FT.CREATE', 'idx', 'SCHEMA', 'Source', 'TAG', 'Version', 'TAG').ok()
messages = [
"AB\x00B", # hex: 41420042
"AB\x00F", # hex: 41420046
]
for i in range(len(messages)):
conn.execute_command(
'HSET', f'doc{i}', 'Source', 'SourceA', 'Message', messages[i],
'Version', 'v1.0')
# Query 1: Basic aggregate with load
query1 = ['FT.AGGREGATE', 'idx', '@Source:{SourceA|SourceB}',
'LOAD', '1', 'Message']
res1 = env.cmd(*query1)
# Query 2: Same query with groupby and reduce tolist
query2 = ['FT.AGGREGATE', 'idx', '@Source:{SourceA|SourceB}',
'LOAD', '1', 'Message',
'GROUPBY', '1', '@Version',
'REDUCE', 'TOLIST', '1', '@Message', 'AS', 'v']
res2 = env.cmd(*query2)
# extract messages from res1
# [1, ['Message', 'AB\x00B'], ['Message', 'AB\x00F']]
list1 = [item[1] for item in res1[1:]]
# extract messages from res2
# [1, ['Version', 'v1.0', 'v', ['AB\x00B', 'AB\x00F']]]
list2 = res2[1][-1]
length1 = len(list1)
length2 = len(list2)
env.assertEqual(
length1, length2,
message=f"Different number of messages: {length1} vs {length2}")
@skip(cluster=False) # This test is only relevant for cluster
def test_mod_14112(env: Env):
'''Test that FT.SEARCH returns an error (not crash) on topology validation failure.
When topology validation fails, the reducer context is NULL. Previously this caused
a SIGSEGV in sendSearchResults. Now we return an error gracefully.'''
# Create an index first (before breaking topology)
env.expect('FT.CREATE', 'idx', 'SCHEMA', 't', 'TEXT').ok()
# Pause topology refresh so our invalid topology stays in effect
env.expect(debug_cmd(), 'PAUSE_TOPOLOGY_UPDATER').ok()
# Set validation timeout to 1ms so that we won't wait for the invalid topology to be validated
env.expect(config_cmd(), 'SET', 'TOPOLOGY_VALIDATION_TIMEOUT', '1').ok()
# Set topology to an invalid one (assuming port 9 is not open)
env.expect('SEARCH.CLUSTERSET',
'MYID',
'1',
'RANGES',
'2',
'SHARD',
'1',
'SLOTRANGE',
'0',
'8191',
'ADDR',
'127.0.0.1:9',
'MASTER',
'SHARD',
'2',
'SLOTRANGE',
'8192',
'16383',
'ADDR',
'127.0.0.1:9',
'MASTER'
).ok()
# Wait for the topology to be applied
wait_for_condition(
lambda: (env.cmd('SEARCH.CLUSTERINFO')[5][0][7] == 9, {}),
'Failed waiting for topology to be applied'
)
# Verify that `FT.SEARCH` queries return an error (not crash)
env.expect('FT.SEARCH', 'idx', '*').error().contains('Could not send query to cluster')