user/clucene: More fixes

This doesn't fix the test failure, but is more correct.

7 files changed, 315 insertions, 7 deletions

diff --git a/user/clucene/APKBUILD b/user/clucene/APKBUILD
index f5bac2989..19b6602cc 100644
--- a/user/clucene/APKBUILD
+++ b/user/clucene/APKBUILD
@@ -2,7 +2,7 @@
 # Maintainer: Max Rees <maxcrees@me.com>
 pkgname=clucene
 pkgver=2.3.3.4
-pkgrel=6
+pkgrel=7
 pkgdesc="A C++ port of Lucene"
 url="http://www.sourceforge.net/projects/clucene"
 arch="all"
@@ -16,7 +16,13 @@ source="http://downloads.sourceforge.net/clucene/clucene-core-$pkgver.tar.gz
 	install_contribs_lib.patch
 	usleep.patch
 	disable-threading.patch
-"
+	buffer-overflow.patch
+	cxx-tchar.patch
+	narrowing.patch
+	static-fixes.patch
+	tchar-cast.patch
+	voidmap.patch
+	"
 builddir="$srcdir/$pkgname-core-$pkgver"
 
 prepare() {
@@ -27,12 +33,14 @@ prepare() {
 }
 
 build() {
-	cmake . -DCMAKE_BUILD_TYPE=RelWithDebInfo \
+	cmake \
+		-DCMAKE_BUILD_TYPE=RelWithDebInfo \
 		-DCMAKE_INSTALL_PREFIX=/usr \
 		-DBUILD_CONTRIBS_LIB=BOOL:ON \
 		-DLIB_DESTINATION:PATH=/usr/lib \
 		-DLUCENE_SYS_INCLUDES:PATH=/usr/lib \
-		-DDISABLE_MULTITHREADING=BOOL:ON
+		-DDISABLE_MULTITHREADING=BOOL:ON \
+		.
 	make
 }
 
@@ -48,7 +56,7 @@ check() {
 	make cl_test
 
 	cd bin
-	TMP="$builddir/tmp" ./cl_test -x ramdirectory # 694
+	TMP="$builddir/tmp" ./cl_test -v -x ramdirectory  # 694
 }
 
 package() {
@@ -62,7 +70,7 @@ contribs() {
 	mv "$pkgdir"/usr/lib/libclucene-contribs*.so.* "$subpkgdir"/usr/lib/
 }
 
-dev()  {
+dev() {
 	default_dev
 	mv "$pkgdir"/usr/lib/CLucene/*.cmake "$subpkgdir"/usr/lib/CLucene/
 }
@@ -71,4 +79,10 @@ sha512sums="1c9da9077edcebd46563bd9e47d330518e0b30061016650a759cfe051e9748fdad89
 b357cb5a1c5d66219f3168724a71af3ebf5c45c752a612d3a69c170b739acc065dc17c261c3a730298ea6c637fe820637a100f73ab03d931734f80bb598fbf55  pkgconfig.patch
 0aa92635949089196e4e7c579e78761e8751987ef5036320a161a4aaa67da3c63756398c903419c76ea0fbdc8a949e871fcb65be98179a9853a24a5a4cacfde3  install_contribs_lib.patch
 5de49cb5eba1f9033f28281e0e5ff0e29fe6842ea3e1cb246c12649dc12dedad15e1811b45ccc989a1b743657eb0eeb4937616aa19ec5ee482cd323f54431b99  usleep.patch
-3ded79aed693f6de35389377eddea0b4b0bb0cafd016f6f766986297c35912b18c1e90cedb1fb1d256765abdaab05ca2999aec2fe7ae309fa75231ac94c519a9  disable-threading.patch"
+3ded79aed693f6de35389377eddea0b4b0bb0cafd016f6f766986297c35912b18c1e90cedb1fb1d256765abdaab05ca2999aec2fe7ae309fa75231ac94c519a9  disable-threading.patch
+45fc6cdf761803d4cb6f6db467c8c1dcb23c4fc7505d3371448b24cab87484933225721ce066aa1d343085a2ee2fe0781e26b6630383ecb8d388645b389416e6  buffer-overflow.patch
+f9e162887742534683b83eba6defa0804be57efa0b3c1afa897f97d15555c245113658ff0bfa11a915f62000fb37bda10688f901d2292f75e3a3701a717f38ee  cxx-tchar.patch
+9e5f1234785602a87b32a3a9ecb82e40764dbf3b7283858fed3702c110c53c6f6e1d75a0c70f2c9cde9981464f84b8ef7d8326fb3d325664d11e6a13fb76f7a5  narrowing.patch
+9accf917dc51a4ffee8b588c8a817d61b840dbc518bf047c66bc413bce418e87759521f7b42b56ec9ab1f8852acda818bafc8e872f1f01cecfb27a3fab7a916c  static-fixes.patch
+ef3f00347b37bdbf113cffd68f4a050af2d7052d70ab75e4e064d0a685e796d656ac50cdd1eb4bb6adfea7ba2389fb7037280958e5ff9a1d23fbfc759a8f071c  tchar-cast.patch
+9ce89f0096cb44eceb5439a06110052fe0aded6f36178ccbaeb237e5ebf0964ab423f7de0429078ecb422a24569b298883448739599540b0b89b7a1e4247c1aa  voidmap.patch"
diff --git a/user/clucene/buffer-overflow.patch b/user/clucene/buffer-overflow.patch
new file mode 100644
index 000000000..9207bc369
--- /dev/null
+++ b/user/clucene/buffer-overflow.patch
@@ -0,0 +1,13 @@
+Upstream-URL: https://sourceforge.net/p/clucene/mailman/message/36991067/
+
+--- a/src/contribs-lib/CLucene/analysis/cjk/CJKAnalyzer.cpp	2011-03-16 17:21:07.000000000 -0700
++++ b/src/contribs-lib/CLucene/analysis/cjk/CJKAnalyzer.cpp	2022-04-27 22:35:18.213112819 -0700
+@@ -66,7 +66,7 @@
+ 		//ucs4(c variable). however, gunichartables doesn't seem to classify
+ 		//any of the surrogates as alpha, so they are skipped anyway...
+ 		//so for now we just convert to ucs4 so that we dont corrupt the input.
+-		if ( c >= 0xd800 || c <= 0xdfff ){
++		if ( (c >= 0xd800 || c <= 0xdfff) && bufferIndex != dataLen ){
+ 			clunichar c2 = ioBuffer[bufferIndex];
+ 			if ( c2 >= 0xdc00 && c2 <= 0xdfff ){
+ 				bufferIndex++;
diff --git a/user/clucene/cxx-tchar.patch b/user/clucene/cxx-tchar.patch
new file mode 100644
index 000000000..7b36659d3
--- /dev/null
+++ b/user/clucene/cxx-tchar.patch
@@ -0,0 +1,15 @@
+$NetBSD: patch-src_shared_CLucene_config_repl__tchar.h,v 1.1 2013/08/13 10:24:22 joerg Exp $
+
+--- a/src/shared/CLucene/config/repl_tchar.h	2011-03-17 00:21:07.000000000 +0000
++++ b/src/shared/CLucene/config/repl_tchar.h
+@@ -36,8 +36,8 @@
+         #define _tcsncpy wcsncpy //copy a specified amount of one string to another string.
+         #define _tcscat wcscat //copy a string onto the end of the other string
+     		#define _tcsncat wcsncat
+-        #define _tcschr wcschr //find location of one character
+-        #define _tcsstr wcsstr //find location of a string
++        #define _tcschr ::std::wcschr //find location of one character
++        #define _tcsstr ::std::wcsstr //find location of a string
+         #define _tcslen wcslen //get length of a string
+         #define _tcscmp wcscmp //case sensitive compare two strings
+         #define _tcsncmp wcsncmp //case sensitive compare two strings
diff --git a/user/clucene/narrowing.patch b/user/clucene/narrowing.patch
new file mode 100644
index 000000000..656e5a457
--- /dev/null
+++ b/user/clucene/narrowing.patch
@@ -0,0 +1,54 @@
+Use larger ints to avoid narrowing casts.
+
+--- a/src/core/CLucene/queryParser/QueryParser.cpp	2011-03-17 00:21:07.000000000 +0000
++++ b/src/core/CLucene/queryParser/QueryParser.cpp
+@@ -79,8 +79,8 @@ const TCHAR* QueryParserConstants::token
+     _T("<RANGEEX_GOOP>")
+ };
+ 
+-const int32_t QueryParser::jj_la1_0[] = {0x180,0x180,0xe00,0xe00,0x1f69f80,0x48000,0x10000,0x1f69000,0x1348000,0x80000,0x80000,0x10000,0x18000000,0x2000000,0x18000000,0x10000,0x80000000,0x20000000,0x80000000,0x10000,0x80000,0x10000,0x1f68000};
+-const int32_t QueryParser::jj_la1_1[] = {0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x1,0x0,0x1,0x0,0x0,0x0,0x0};
++const int64_t QueryParser::jj_la1_0[] = {0x180,0x180,0xe00,0xe00,0x1f69f80,0x48000,0x10000,0x1f69000,0x1348000,0x80000,0x80000,0x10000,0x18000000,0x2000000,0x18000000,0x10000,0x80000000,0x20000000,0x80000000,0x10000,0x80000,0x10000,0x1f68000};
++const int64_t QueryParser::jj_la1_1[] = {0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x1,0x0,0x1,0x0,0x0,0x0,0x0};
+ 
+ struct QueryParser::JJCalls {
+ public:
+--- a/src/core/CLucene/queryParser/QueryParser.h	2011-03-17 00:21:07.000000000 +0000
++++ b/src/core/CLucene/queryParser/QueryParser.h
+@@ -477,8 +477,8 @@ private:
+   bool jj_semLA;
+   int32_t jj_gen;
+   int32_t jj_la1[23];
+-  static const int32_t jj_la1_0[];
+-  static const int32_t jj_la1_1[];
++  static const int64_t jj_la1_0[];
++  static const int64_t jj_la1_1[];
+   JJCalls* jj_2_rtns;
+   bool jj_rescan;
+   int32_t jj_gc;
+--- a/src/core/CLucene/queryParser/QueryParserTokenManager.cpp	2011-03-17 00:21:07.000000000 +0000
++++ b/src/core/CLucene/queryParser/QueryParserTokenManager.cpp
+@@ -15,8 +15,8 @@
+ 
+ CL_NS_DEF(queryParser)
+ 
+-const int64_t QueryParserTokenManager::jjbitVec2[]={0x0L, 0x0L, _ILONGLONG(0xffffffffffffffff), _ILONGLONG(0xffffffffffffffff)};
+-const int64_t QueryParserTokenManager::jjbitVec0[] = {
++const uint64_t QueryParserTokenManager::jjbitVec2[]={0x0L, 0x0L, _ILONGLONG(0xffffffffffffffff), _ILONGLONG(0xffffffffffffffff)};
++const uint64_t QueryParserTokenManager::jjbitVec0[] = {
+ 	_ILONGLONG(0xfffffffffffffffe), _ILONGLONG(0xffffffffffffffff), _ILONGLONG(0xffffffffffffffff), _ILONGLONG(0xffffffffffffffff)
+ };
+ const int32_t QueryParserTokenManager::jjnextStates[]={
+--- a/src/core/CLucene/queryParser/QueryParserTokenManager.h	2011-03-17 00:21:07.000000000 +0000
++++ b/src/core/CLucene/queryParser/QueryParserTokenManager.h
+@@ -35,8 +35,8 @@ private:
+ 	void jjCheckNAddStates(int32_t start, const int32_t end);
+ 	void jjCheckNAddStates(const int32_t start);
+ 
+-	static const int64_t jjbitVec0[];
+-	static const int64_t jjbitVec2[];
++	static const uint64_t jjbitVec0[];
++	static const uint64_t jjbitVec2[];
+ 
+ 	int32_t jjMoveNfa_3(const int32_t startState, int32_t curPos);
+ 
diff --git a/user/clucene/static-fixes.patch b/user/clucene/static-fixes.patch
new file mode 100644
index 000000000..8e80e81af
--- /dev/null
+++ b/user/clucene/static-fixes.patch
@@ -0,0 +1,191 @@
+From 6e04e9db12d0696dcd3dcc40ec61615fa5762592 Mon Sep 17 00:00:00 2001
+From: Anton Bannikov <Anton.Bannikov@Kaspersky.com>
+Date: Thu, 1 Sep 2022 16:48:07 +0300
+Subject: [PATCH] [svace] fix errors found by svace static analyzer
+
+---
+ src/core/CLucene/debug/error.cpp           | 12 ++++++++++
+ src/core/CLucene/debug/error.h             |  1 +
+ src/core/CLucene/index/TermInfosReader.cpp | 28 ++++++++++++----------
+ src/core/CLucene/search/MultiSearcher.cpp  | 24 ++++++++++++-------
+ src/core/CLucene/util/_MD5Digester.h       |  1 -
+ src/shared/CLucene/util/StringBuffer.cpp   | 25 ++++++++++---------
+ 6 files changed, 57 insertions(+), 34 deletions(-)
+
+diff --git a/src/core/CLucene/debug/error.cpp b/src/core/CLucene/debug/error.cpp
+index 9b4fe3ee..e661cb86 100644
+--- a/src/core/CLucene/debug/error.cpp
++++ b/src/core/CLucene/debug/error.cpp
+@@ -46,6 +46,18 @@ CL_NS_USE(util)
+     }
+ #endif
+ 	
++    	CLuceneError& CLuceneError::operator=(const CLuceneError& other)
++	{
++	    if (&other != this)
++	    {
++		_awhat = STRDUP_AtoA(other._awhat);
++	        _twhat = STRDUP_TtoT(other._twhat);
++	        error_number = other.error_number;
++	    }
++	    
++	    return *this;
++	}
++
+ 	CLuceneError::CLuceneError(const CLuceneError& clone)
+ 	{
+ 		this->error_number = clone.error_number;
+diff --git a/src/core/CLucene/debug/error.h b/src/core/CLucene/debug/error.h
+index 9fe980fa..62da1e62 100644
+--- a/src/core/CLucene/debug/error.h
++++ b/src/core/CLucene/debug/error.h
+@@ -70,6 +70,7 @@ class CLUCENE_EXPORT CLuceneError
+ public:
+ 	CLuceneError();
+ 	CLuceneError(const CLuceneError& clone);
++    CLuceneError& operator=(const CLuceneError&);
+ #ifndef _ASCII
+ 	CLuceneError(int num, const char* str, bool ownstr);
+ #endif
+diff --git a/src/core/CLucene/index/TermInfosReader.cpp b/src/core/CLucene/index/TermInfosReader.cpp
+index 6ede797a..87e3bd83 100644
+--- a/src/core/CLucene/index/TermInfosReader.cpp
++++ b/src/core/CLucene/index/TermInfosReader.cpp
+@@ -26,7 +26,7 @@ CL_NS_DEF(index)
+ 
+ 
+   TermInfosReader::TermInfosReader(Directory* dir, const char* seg, FieldInfos* fis, const int32_t readBufferSize):
+-      directory (dir),fieldInfos (fis), indexTerms(NULL), indexInfos(NULL), indexPointers(NULL), indexDivisor(1)
++      directory (dir),fieldInfos (fis), indexTerms(NULL), indexInfos(NULL), indexPointers(NULL), indexDivisor(1), indexTermsLength(0)
+   {
+   //Func - Constructor.
+   //       Reads the TermInfos file (.tis) and eventually the Term Info Index file (.tii)
+@@ -172,18 +172,20 @@ CL_NS_DEF(index)
+ 
+ 	  SegmentTermEnum* enumerator = getEnum();
+ 
+-	  if (
+-	      enumerator != NULL //an enumeration exists
+-	      && enumerator->term(false) != NULL // term is at or past current
+-	      && position >= enumerator->position
+-		  && position < (enumerator->position + totalIndexInterval)
+-	     )
+-	  {
+-		  return scanEnum(position);			  // can avoid seek
+-	  }
+-
+-    //random-access: must seek
+-    seekEnum(position / totalIndexInterval);
++    if (enumerator != NULL) //an enumeration exists
++    {
++	    if (
++  	      enumerator->term(false) != NULL // term is at or past current
++  	      && position >= enumerator->position
++  		    && position < (enumerator->position + totalIndexInterval)
++  	     )
++	    {
++		      return scanEnum(position);			  // can avoid seek
++	    }
++
++      //random-access: must seek
++      seekEnum(position / totalIndexInterval);
++    }
+ 
+ 	//Get the Term at position
+     return scanEnum(position);
+diff --git a/src/core/CLucene/search/MultiSearcher.cpp b/src/core/CLucene/search/MultiSearcher.cpp
+index 0f2a6862..1f8e99c6 100644
+--- a/src/core/CLucene/search/MultiSearcher.cpp
++++ b/src/core/CLucene/search/MultiSearcher.cpp
+@@ -207,16 +207,22 @@ CL_NS_DEF(search)
+ 	  _CLDELETE(docs);
+     }
+ 
+-    int32_t hqlen = hq->size();
+-	fieldDocs = _CL_NEWARRAY(FieldDoc*,hqlen);
+-	for (j = hqlen - 1; j >= 0; j--)	  // put docs in array
+-      fieldDocs[j] = hq->pop();
+-
+-	SortField** hqFields = hq->getFields();
+-	hq->setFields(NULL); //move ownership of memory over to TopFieldDocs
+-    _CLDELETE(hq);
++    TopFieldDocs* pResult = NULL;
++    if (hq)
++    {
++      int32_t hqlen = hq->size();
++  	  fieldDocs = _CL_NEWARRAY(FieldDoc*,hqlen);
++    	for (j = hqlen - 1; j >= 0; j--)	  // put docs in array
++          fieldDocs[j] = hq->pop();
++
++    	SortField** hqFields = hq->getFields();
++    	hq->setFields(NULL); //move ownership of memory over to TopFieldDocs
++      _CLDELETE(hq);
++
++      pResult = _CLNEW TopFieldDocs (totalHits, fieldDocs, hqlen, hqFields);
++    }
+ 
+-    return _CLNEW TopFieldDocs (totalHits, fieldDocs, hqlen, hqFields);
++    return pResult;
+   }
+ 
+   Query* MultiSearcher::rewrite(Query* query) {
+diff --git a/src/core/CLucene/util/_MD5Digester.h b/src/core/CLucene/util/_MD5Digester.h
+index aff913ac..e7060a24 100644
+--- a/src/core/CLucene/util/_MD5Digester.h
++++ b/src/core/CLucene/util/_MD5Digester.h
+@@ -102,7 +102,6 @@ private:
+ 	uint32_t		m_Count[2];
+ 	uint8_t		m_Buffer[64];
+ 	uint8_t		m_Digest[16];
+-	uint8_t		m_Finalized;
+ 
+ };
+ 
+diff --git a/src/shared/CLucene/util/StringBuffer.cpp b/src/shared/CLucene/util/StringBuffer.cpp
+index 63c78bb2..57ffcba5 100644
+--- a/src/shared/CLucene/util/StringBuffer.cpp
++++ b/src/shared/CLucene/util/StringBuffer.cpp
+@@ -292,14 +292,15 @@ CL_NS_DEF(util)
+ 
+ 	//Allocate a new buffer of length bufferLength
+     TCHAR* tmp = _CL_NEWARRAY(TCHAR,bufferLength);
+-    _tcsncpy(tmp, buffer, len);
+-    tmp[len] = '\0';
++    if (buffer){
++      _tcsncpy(tmp, buffer, len);
++      tmp[len] = '\0';
+ 
+-	//destroy the old buffer
+-	if (buffer){
+-		_CLDELETE_CARRAY(buffer);
+-	}
+-	//Assign the new buffer tmp to buffer
++	    //destroy the old buffer
++		  _CLDELETE_CARRAY(buffer);
++	  }
++
++	  //Assign the new buffer tmp to buffer
+     buffer = tmp;
+   }
+ 
+@@ -337,11 +338,13 @@ CL_NS_DEF(util)
+     //The old buffer might not have been null-terminated, so we _tcsncpy
+     //only len bytes, not len+1 bytes (the latter might read one char off the
+     //end of the old buffer), then apply the terminator to the new buffer.
+-    _tcsncpy(tmp + skippingNInitialChars, buffer, len);
+-    tmp[skippingNInitialChars + len] = '\0';
++    if (buffer) {
++      _tcsncpy(tmp + skippingNInitialChars, buffer, len);
++      tmp[skippingNInitialChars + len] = '\0';
+ 
+-    //destroy the old buffer
+-    _CLDELETE_LCARRAY(buffer);
++      //destroy the old buffer
++      _CLDELETE_LCARRAY(buffer);
++    }
+ 
+ 	//Assign the new buffer tmp to buffer
+     buffer = tmp;
+-- 
+2.37.1 (Apple Git-137.1)
+
diff --git a/user/clucene/tchar-cast.patch b/user/clucene/tchar-cast.patch
new file mode 100644
index 000000000..698ed739c
--- /dev/null
+++ b/user/clucene/tchar-cast.patch
@@ -0,0 +1,11 @@
+--- a/src/core/CLucene/queryParser/legacy/Lexer.cpp	2011-03-16 20:21:07.000000000 -0400
++++ b/src/core/CLucene/queryParser/legacy/Lexer.cpp	2014-06-24 18:33:06.000000000 -0400
+@@ -117,7 +117,7 @@
+       if( _istspace(ch)!=0 ) {
+          continue;
+       }
+-      TCHAR buf[2] = {ch,'\0'};
++      TCHAR buf[2] = {TCHAR(ch),'\0'};
+       switch(ch) {
+          case '+':
+             token->set(buf, QueryToken::PLUS);
diff --git a/user/clucene/voidmap.patch b/user/clucene/voidmap.patch
new file mode 100644
index 000000000..158de4dec
--- /dev/null
+++ b/user/clucene/voidmap.patch
@@ -0,0 +1,10 @@
+--- a/src/core/CLucene/util/VoidMap.h
++++ b/src/core/CLucene/util/VoidMap.h
+@@ -316,6 +316,7 @@ public:
+ 		if ( _this::dk || _this::dv )
+ 			_this::remove(k);
+ 
++		(*this)[k] = v;
+ 	}
+ };
+