Merge pull request #72 from leanote/feature-full-text-query

Feature full text query on Android

app/src/main/java/org/houxg/leamonax/background/NoteSyncService.java

@@ -63,6 +63,7 @@ public class NoteSyncService extends Service {
                         if (!subscriber.isUnsubscribed()) {
                             NoteService.fetchFromServer();
                             NoteService.pushToServer();
+                            NoteService.buildFTSNote();
                             subscriber.onNext(null);
                             subscriber.onCompleted();
                         }

app/src/main/java/org/houxg/leamonax/database/AppDataBase.java

@@ -19,7 +19,7 @@ import org.houxg.leamonax.model.RelationshipOfNoteTag;
 import org.houxg.leamonax.model.Tag;
 import org.houxg.leamonax.model.Tag_Table;
 
-@Database(name = "leanote_db", version = 4)
+@Database(name = "leanote_db", version = 5)
 public class AppDataBase {
 
     @Migration(version = 2, database = AppDataBase.class)
@@ -157,4 +157,12 @@ public class AppDataBase {
             cursor.close();
         }
     }
+
+    @Migration(version = 5, database = AppDataBase.class)
+    public static class UpdateNoteContentToFTS extends BaseMigration {
+        @Override
+        public void migrate(DatabaseWrapper database) {
+            database.execSQL("CREATE VIRTUAL TABLE fts_note USING fts4 (content='note', content)");
+        }
+    }
 }

app/src/main/java/org/houxg/leamonax/database/NoteDataStore.java

@@ -1,10 +1,14 @@
 package org.houxg.leamonax.database;
 
 
+import android.database.Cursor;
+
+import com.raizlabs.android.dbflow.config.FlowManager;
 import com.raizlabs.android.dbflow.sql.language.Join;
 import com.raizlabs.android.dbflow.sql.language.NameAlias;
 import com.raizlabs.android.dbflow.sql.language.SQLite;
 import com.raizlabs.android.dbflow.sql.language.property.IProperty;
+import com.raizlabs.android.dbflow.structure.database.DatabaseWrapper;
 
 import org.houxg.leamonax.model.Account;
 import org.houxg.leamonax.model.Note;
@@ -16,8 +20,10 @@ import org.houxg.leamonax.model.Tag;
 import org.houxg.leamonax.model.Tag_Table;
 
 import java.util.ArrayList;
+import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Locale;
+import java.util.Set;
 
 public class NoteDataStore {
     public static List<Note> searchByTitle(String keyword) {
@@ -31,6 +37,80 @@ public class NoteDataStore {
                 .queryList();
     }
 
+    public static void updateFTSNoteByLocalId(Long localId) {
+        Note note = getByLocalId(localId);
+        DatabaseWrapper databaseWrapper = FlowManager.getWritableDatabase(AppDataBase.class);
+        String query = "UPDATE fts_note SET content = '" + note.getContent() + "' where rowid = " + localId;
+        databaseWrapper.execSQL(query);
+    }
+
+    public static boolean isExistsTableFTSNote() {
+        boolean result = false;
+        DatabaseWrapper databaseWrapper = FlowManager.getWritableDatabase(AppDataBase.class);
+        String query = "select count(*) as c from sqlite_master where type ='table' and name ='fts_note'";
+        Cursor cursor = databaseWrapper.rawQuery(query, null);
+        if(cursor.moveToNext()){
+            int count = cursor.getInt(0);
+            if(count > 0){
+                result = true;
+            }
+        }
+        return result;
+    }
+
+    public static void createTableFTSNote() {
+        DatabaseWrapper databaseWrapper = FlowManager.getWritableDatabase(AppDataBase.class);
+        String query = "CREATE VIRTUAL TABLE fts_note USING fts4 (content='note', content)";
+        databaseWrapper.execSQL(query);
+    }
+
+    public static void FTSNoteRebuild() {
+        if (!isExistsTableFTSNote()) {
+            createTableFTSNote();
+        }
+        FTSNoteRebuildInternal();
+    }
+
+    public static void FTSNoteRebuildInternal() {
+        DatabaseWrapper databaseWrapper = FlowManager.getWritableDatabase(AppDataBase.class);
+        String query = "INSERT INTO fts_note(fts_note) VALUES('rebuild')";//This can be slow
+        databaseWrapper.execSQL(query);
+    }
+
+    public static List<Note> searchByKeyword(String keyword) {
+        if (!isExistsTableFTSNote()) {
+            createTableFTSNote();
+            return searchByTitle(keyword);
+        } else {
+            return searchByFullTextSearch(keyword);
+        }
+
+    }
+    public static List<Note> searchByFullTextSearch(String keyword) {
+        Set<Long> set = new LinkedHashSet<>();
+        DatabaseWrapper databaseWrapper = FlowManager.getWritableDatabase(AppDataBase.class);
+        String query = "select id from note where userid = ? and istrash = 0 and isdeleted = 0 and id in " +
+                "(select rowid from fts_note where fts_note match ?)";////查询Content中满足条件的记录
+        Cursor cursor = databaseWrapper.rawQuery(query, new String[]{Account.getCurrent().getUserId(), "*" + keyword + "*"});
+        while(cursor.moveToNext()) {
+            set.add(cursor.getLong(cursor.getColumnIndex("id")));
+        }
+        cursor.close();
+
+        query = "select id from note where userid = ? and istrash = 0 and isdeleted = 0 and title like ?";//查询title中满足条件的记录
+        cursor = databaseWrapper.rawQuery(query, new String[]{Account.getCurrent().getUserId(), "%" + keyword + "%"});//查询Content中满足条件的记录
+        while(cursor.moveToNext()) {
+            set.add(cursor.getLong(cursor.getColumnIndex("id")));
+        }
+        cursor.close();
+
+        return SQLite.select()
+                .from(Note.class)
+                .where(Note_Table.id.in(set))
+                .queryList();
+    }
+
+
     public static Note getByServerId(String serverId) {
         return SQLite.select()
                 .from(Note.class)

app/src/main/java/org/houxg/leamonax/service/NoteService.java

@@ -24,11 +24,13 @@ import org.houxg.leamonax.model.UpdateRe;
 import org.houxg.leamonax.network.ApiProvider;
 import org.houxg.leamonax.utils.CollectionUtils;
 import org.houxg.leamonax.utils.RetrofitUtils;
+import org.houxg.leamonax.utils.SharedPreferenceUtils;
 import org.houxg.leamonax.utils.StringUtils;
 import org.houxg.leamonax.utils.TimeUtils;
 
 import java.io.File;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Locale;
@@ -38,7 +40,6 @@ import okhttp3.MediaType;
 import okhttp3.MultipartBody;
 import okhttp3.RequestBody;
 import retrofit2.Call;
-import rx.Observable;
 
 public class NoteService {
 
@@ -48,6 +49,8 @@ public class NoteService {
     private static final String MULTIPART_FORM_DATA = "multipart/form-data";
     private static final String CONFLICT_SUFFIX = "--conflict";
     private static final int MAX_ENTRY = 20;
+    public static final String SP_HAS_FTS_FULL_BUILD = "sp_has_fts_full_build";
+    public static final String SP_FTS_INCREASE_BUILD_KES = "sp_has_increase_build_KEYS";
 
     public static void pushToServer() {
         List<Note> notes = NoteDataStore.getAllDirtyNotes(Account.getCurrent().getUserId());
@@ -57,6 +60,31 @@ public class NoteService {
             }
         }
     }
+    public static void buildFTSNote() {
+        if (!SharedPreferenceUtils.read(SharedPreferenceUtils.CONFIG, SP_HAS_FTS_FULL_BUILD, false)) {//存在缺馅，对于多账户用户而言，每个账户第一次全量同步都会触发rebuild
+            NoteDataStore.FTSNoteRebuild();
+            SharedPreferenceUtils.write(SharedPreferenceUtils.CONFIG, SP_HAS_FTS_FULL_BUILD, true);
+            SharedPreferenceUtils.write(SharedPreferenceUtils.CONFIG, SP_FTS_INCREASE_BUILD_KES, "");
+        } else {
+            String noteLocalIds = SharedPreferenceUtils.read(SharedPreferenceUtils.CONFIG, SP_FTS_INCREASE_BUILD_KES, "");
+            String array[] = TextUtils.split(noteLocalIds, ",");
+            for (String localIdStr: array) {
+                Long localId = Long.valueOf(localIdStr);
+                NoteDataStore.updateFTSNoteByLocalId(localId);
+            }
+            SharedPreferenceUtils.write(SharedPreferenceUtils.CONFIG, SP_FTS_INCREASE_BUILD_KES, "");
+        }
+    }
+
+    public static void addInCreaseBuildKey(Long localId) {
+        String noteLocalIds = SharedPreferenceUtils.read(SharedPreferenceUtils.CONFIG, SP_FTS_INCREASE_BUILD_KES, "");
+        String array[] = TextUtils.split(noteLocalIds, ",");
+        List<String> list = new ArrayList<>(Arrays.asList(array));
+        if (!list.contains(String.valueOf(localId))) {
+            list.add(String.valueOf(localId));
+        }
+        SharedPreferenceUtils.write(SharedPreferenceUtils.CONFIG, SP_FTS_INCREASE_BUILD_KES, TextUtils.join(",", list));
+    }
 
     public static void fetchFromServer() {
         //sync notebook

app/src/main/java/org/houxg/leamonax/ui/NoteFragment.java

@@ -163,7 +163,7 @@ public class NoteFragment extends Fragment implements NoteAdapter.NoteAdapterLis
                 notes = NoteDataStore.getByTagText(mode.tagText, Account.getCurrent().getUserId());
                 break;
             case SEARCH:
-                notes = NoteDataStore.searchByTitle(mode.keywords);
+                notes = NoteDataStore.searchByKeyword(mode.keywords);
                 mNoteList.setHighlight(mode.keywords);
                 break;
             default:

app/src/main/java/org/houxg/leamonax/ui/SearchActivity.java

@@ -33,15 +33,15 @@ public class SearchActivity extends BaseActivity {
         mSearchView.setOnQueryTextListener(new SearchView.OnQueryTextListener() {
             @Override
             public boolean onQueryTextSubmit(String query) {
-                return false;
+                NoteFragment.Mode mode = NoteFragment.Mode.SEARCH;
+                mode.setKeywords(query);
+                mNoteFragment.setMode(mode);
+                return true;
             }
 
             @Override
             public boolean onQueryTextChange(String newText) {
-                NoteFragment.Mode mode = NoteFragment.Mode.SEARCH;
+                return false;
-                mode.setKeywords(newText);
-                mNoteFragment.setMode(mode);
-                return true;
             }
         });
 

app/src/main/java/org/houxg/leamonax/ui/SettingsActivity.java

@@ -2,6 +2,7 @@ package org.houxg.leamonax.ui;
 
 
 import android.app.AlertDialog;
+import android.app.ProgressDialog;
 import android.content.DialogInterface;
 import android.content.Intent;
 import android.os.Bundle;
@@ -22,11 +23,10 @@ import org.houxg.leamonax.database.NoteTagDataStore;
 import org.houxg.leamonax.database.NotebookDataStore;
 import org.houxg.leamonax.model.Account;
 import org.houxg.leamonax.model.BaseResponse;
-import org.houxg.leamonax.model.Note;
-import org.houxg.leamonax.model.Notebook;
-import org.houxg.leamonax.model.RelationshipOfNoteTag;
 import org.houxg.leamonax.model.Tag;
 import org.houxg.leamonax.service.AccountService;
+import org.houxg.leamonax.service.NoteService;
+import org.houxg.leamonax.utils.SharedPreferenceUtils;
 import org.houxg.leamonax.utils.ToastUtils;
 
 import java.util.List;
@@ -41,6 +41,8 @@ import rx.android.schedulers.AndroidSchedulers;
 import rx.functions.Action1;
 import rx.schedulers.Schedulers;
 
+import static org.houxg.leamonax.service.NoteService.SP_HAS_FTS_FULL_BUILD;
+
 public class SettingsActivity extends BaseActivity {
 
     private final String[] mEditors = new String[]{"RichText", "Markdown"};
@@ -59,6 +61,9 @@ public class SettingsActivity extends BaseActivity {
     TextView mHostTv;
     @BindView(R.id.ll_clear)
     View mClearDataView;
+    @BindView(R.id.ll_fts_rebuild)
+    View mFtsRebuildLayout;
+    private ProgressDialog mDialog;
 
     @Override
     protected void onCreate(@Nullable Bundle savedInstanceState) {
@@ -146,8 +151,8 @@ public class SettingsActivity extends BaseActivity {
     @OnClick(R.id.ll_change_password)
     void clickedPassword() {
         View view = LayoutInflater.from(this).inflate(R.layout.dialog_change_passowrd, null);
-        final EditText mOldPasswordEt = (EditText) view.findViewById(R.id.et_old_password);
+        final EditText mOldPasswordEt = view.findViewById(R.id.et_old_password);
-        final EditText mNewPasswordEt = (EditText) view.findViewById(R.id.et_new_password);
+        final EditText mNewPasswordEt = view.findViewById(R.id.et_new_password);
         new AlertDialog.Builder(this)
                 .setTitle(R.string.change_password)
                 .setView(view)
@@ -183,6 +188,52 @@ public class SettingsActivity extends BaseActivity {
                 .show();
     }
 
+    private void showProgressDialog() {
+        if (mDialog == null) {
+            mDialog = new ProgressDialog(this);
+            mDialog.setTitle(R.string.progress_dialog_loading_msg);
+            mDialog.setCancelable(false);
+            mDialog.show();
+        }
+    }
+
+    private void hideProgressDialog() {
+        if (mDialog != null) {
+            mDialog.dismiss();
+            mDialog = null;
+        }
+    }
+
+    @OnClick(R.id.ll_fts_rebuild)
+    void rebuildIndex() {
+        new AlertDialog.Builder(this)
+                .setTitle(R.string.full_text_search_index_rebuild)
+                .setMessage(R.string.full_text_search_rebuild_index_msg)
+                .setNegativeButton(R.string.no, new DialogInterface.OnClickListener() {
+                    @Override
+                    public void onClick(DialogInterface dialog, int which) {
+                        dialog.dismiss();
+                    }
+                })
+                .setPositiveButton(R.string.yes, new DialogInterface.OnClickListener() {
+                    @Override
+                    public void onClick(DialogInterface dialog, int which) {
+                        showProgressDialog();
+                        SharedPreferenceUtils.write(SharedPreferenceUtils.CONFIG, SP_HAS_FTS_FULL_BUILD, false);
+                        NoteService.buildFTSNote();
+                        dialog.dismiss();
+                        mFtsRebuildLayout.postDelayed(new Runnable() {
+                            @Override
+                            public void run() {
+                                hideProgressDialog();
+                                ToastUtils.show(SettingsActivity.this, R.string.full_text_search_index_rebuild_success);
+                            }
+                        }, 1000 * 15);
+                    }
+                })
+                .show();
+    }
+
     private void clearData() {
         Observable.create(
                 new Observable.OnSubscribe<Void>() {

app/src/main/java/org/houxg/leamonax/ui/edit/NoteEditActivity.java

@@ -169,6 +169,7 @@ public class NoteEditActivity extends BaseActivity implements EditorFragment.Edi
                             @Override
                             public void call(Wrapper wrapper) {
                                 saveAsDraft(wrapper);
+                                NoteService.addInCreaseBuildKey(wrapper.note.getId());
                                 setResult(RESULT_OK);
                                 NetworkUtils.checkNetwork();
                             }
@@ -240,6 +241,7 @@ public class NoteEditActivity extends BaseActivity implements EditorFragment.Edi
                                         wrapper.note.delete();
                                     } else {
                                         saveAsDraft(wrapper);
+                                        NoteService.addInCreaseBuildKey(wrapper.note.getId());
                                     }
                                 }
                             });

app/src/main/java/org/houxg/leamonax/utils/SharedPreferenceUtils.java

@@ -5,10 +5,11 @@ import android.content.Context;
 import android.content.SharedPreferences;
 
 import org.houxg.leamonax.Leamonax;
+import org.houxg.leamonax.model.Account;
 
 public class SharedPreferenceUtils {
 
-    public static final String CONFIG = "CONFIG";
+    public static final String CONFIG = "CONFIG_" + Account.getCurrent().getUserId();
 
     public static SharedPreferences getSharedPreferences(String name) {
         return Leamonax.getContext().getSharedPreferences(name, Context.MODE_PRIVATE);

app/src/main/res/layout/activity_settings.xml

@@ -139,6 +139,28 @@
 
             <include layout="@layout/divider" />
 
+            <LinearLayout
+                android:id="@+id/ll_fts_rebuild"
+                style="@style/SettingsPanel"
+                android:orientation="vertical">
+
+                <TextView
+                    style="@style/SettingsSecondaryText"
+                    android:layout_width="wrap_content"
+                    android:layout_height="wrap_content"
+                    android:text="@string/full_text_search" />
+
+                <TextView
+                    android:id="@+id/tv_rebuild"
+                    style="@style/SettingsStatus"
+                    android:layout_width="wrap_content"
+                    android:layout_height="wrap_content"
+                    android:text="@string/full_text_search_index_rebuild" />
+
+            </LinearLayout>
+
+            <include layout="@layout/divider" />
+
             <LinearLayout
                 android:id="@+id/ll_image_size"
                 style="@style/SettingsPanel"

app/src/main/res/values-zh/strings.xml

@@ -122,4 +122,9 @@
     <string name="lea_api_error_need_upgrade_account">需要升级蚂蚁笔记账户</string>
     <string name="webview_select_picture">选择图片</string>
     <string name="note_not_load_completed">笔记未加载完，不能保存</string>
+    <string name="full_text_search">全文搜索</string>
+    <string name="full_text_search_index_rebuild">重建索引</string>
+    <string name="full_text_search_index_rebuild_success">索引构建成功</string>
+    <string name="full_text_search_rebuild_index_msg">这是一个很耗时的操作,你确定要重新构建索引么(默认情况下我们不推荐用户使用此功能)?</string>
+    <string name="progress_dialog_loading_msg">Processing&#8230;</string>
 </resources>

app/src/main/res/values/strings.xml

@@ -124,4 +124,9 @@
     <string name="lea_api_error_need_upgrade_account">need upgrade Leanote account</string>
     <string name="webview_select_picture">Select Picture</string>
     <string name="note_not_load_completed">Notes are not loaded and cannot be saved</string>
+    <string name="full_text_search">Full text search</string>
+    <string name="full_text_search_index_rebuild">rebuild fts index</string>
+    <string name="full_text_search_index_rebuild_success">Index build succeeded</string>
+    <string name="full_text_search_rebuild_index_msg">This is a time-consuming operation. Are you sure you want to rebuild the index(We don\'t recommend users to use this feature by default)?</string>
+    <string name="progress_dialog_loading_msg">Loading&#8230;</string>
 </resources>