drgreen/android_frameworks_base
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Return count of rows in a resultset only once (when startPos = 0)

Browse Source 
If a query returns 100 rows and say only 10 rows fit in 1MB, then client
receiving the cursor from the ContentProvider needs to paginate.
ContentProvider returns count of total data everytime it returns a page
(= 1MB) of data to the client.
Returning total count causes reading (and skipping unwanted) data
from sqlite.
Instead, it should be sufficient to get total count once
and re-use the count value during the life of the cursor
until a requery is performed on the cursor.
(Count won't change unless data is changed - in which case
the cursor is asked to perform requery anyway. So doing count
once and reusing it should work)
Change-Id: I3520d94524dda07be9bcff56b6fbae5276af1d3b
This commit is contained in:
Vasu Nori
2010-08-12 18:16:35 -07:00
parent a78eca95ff
commit b18f27dbf4
4 changed files with 171 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files

39
core/java/android/database/sqlite/SQLiteCursor.java
View File

@ -56,7 +56,7 @@ public class SQLiteCursor extends AbstractWindowedCursor {
private final SQLiteCursorDriver mDriver;
/** The number of rows in the cursor */
private int mCount = NO_COUNT;
private volatile int mCount = NO_COUNT;
/** A mapping of column names to column indices, to speed up lookups */
private Map<String, Integer> mColumnNameMap;
@ -138,13 +138,21 @@ public class SQLiteCursor extends AbstractWindowedCursor {
}
try {
int count = getQuery().fillWindow(cw, mMaxRead, mCount);
// return -1 means not finished
// return -1 means there is still more data to be retrieved from the resultset
if (count != 0) {
if (count == NO_COUNT){
mCount += mMaxRead;
if (Log.isLoggable(TAG, Log.DEBUG)) {
Log.d(TAG, "received -1 from native_fill_window. read " +
mCount + " rows so far");
}
sendMessage();
} else {
mCount = count;
mCount += count;
if (Log.isLoggable(TAG, Log.DEBUG)) {
Log.d(TAG, "received all data from native_fill_window. read " +
mCount + " rows.");
}
sendMessage();
break;
}
@ -308,13 +316,23 @@ public class SQLiteCursor extends AbstractWindowedCursor {
}
}
mWindow.setStartPosition(startPos);
mCount = getQuery().fillWindow(mWindow, mInitialRead, 0);
// return -1 means not finished
if (mCount == NO_COUNT){
int count = getQuery().fillWindow(mWindow, mInitialRead, 0);
// return -1 means there is still more data to be retrieved from the resultset
if (count == NO_COUNT){
mCount = startPos + mInitialRead;
if (Log.isLoggable(TAG, Log.DEBUG)) {
Log.d(TAG, "received -1 from native_fill_window. read " + mCount + " rows so far");
}
Thread t = new Thread(new QueryThread(mCursorState), "query thread");
t.start();
}
} else if (startPos == 0) { // native_fill_window returns count(*) only for startPos = 0
if (Log.isLoggable(TAG, Log.DEBUG)) {
Log.d(TAG, "received count(*) from native_fill_window: " + count);
}
mCount = count;
} else if (mCount <= 0) {
throw new IllegalStateException("count should never be non-zero negative number");
}
}
private synchronized SQLiteQuery getQuery() {
@ -504,4 +522,11 @@ public class SQLiteCursor extends AbstractWindowedCursor {
super.finalize();
}
}
/**
* this is only for testing purposes.
*/
/* package */ int getMCount() {
return mCount;
}
}

9
core/java/android/database/sqlite/SQLiteQuery.java
View File

@ -18,6 +18,7 @@ package android.database.sqlite;
import android.database.CursorWindow;
import android.os.SystemClock;
import android.util.Log;
/**
* A SQLite program that represents a query that reads the resulting rows into a CursorWindow.
@ -58,6 +59,7 @@ public class SQLiteQuery extends SQLiteProgram {
/* package */ SQLiteQuery(SQLiteDatabase db, SQLiteQuery query) {
super(db, query.mSql);
this.mBindArgs = query.mBindArgs;
this.mOffsetIndex = query.mOffsetIndex;
}
/**
@ -78,8 +80,8 @@ public class SQLiteQuery extends SQLiteProgram {
// if the start pos is not equal to 0, then most likely window is
// too small for the data set, loading by another thread
// is not safe in this situation. the native code will ignore maxRead
int numRows = native_fill_window(window, window.getStartPosition(), mOffsetIndex,
maxRead, lastPos);
int numRows = native_fill_window(window, window.getStartPosition(),
mOffsetIndex, maxRead, lastPos);
mDatabase.logTimeStat(mSql, timeStart);
return numRows;
} catch (IllegalStateException e){
@ -88,6 +90,9 @@ public class SQLiteQuery extends SQLiteProgram {
} catch (SQLiteDatabaseCorruptException e) {
mDatabase.onCorruption();
throw e;
} catch (SQLiteException e) {
Log.e(TAG, "exception: " + e.getMessage() + "; query: " + mSql);
throw e;
} finally {
window.releaseReference();
}

46
core/jni/android_database_SQLiteQuery.cpp
View File

@ -15,7 +15,7 @@
*/
#undef LOG_TAG
#define LOG_TAG "Cursor"
#define LOG_TAG "SqliteCursor.cpp"
#include <jni.h>
#include <JNIHelp.h>
@ -116,6 +116,7 @@ static jint native_fill_window(JNIEnv* env, jobject object, jobject javaWindow,
int retryCount;
int boundParams;
CursorWindow * window;
bool gotAllRows = true;
if (statement == NULL) {
LOGE("Invalid statement in fillWindow()");
@ -131,8 +132,7 @@ static jint native_fill_window(JNIEnv* env, jobject object, jobject javaWindow,
err = sqlite3_bind_int(statement, offsetParam, startPos);
if (err != SQLITE_OK) {
LOGE("Unable to bind offset position, offsetParam = %d", offsetParam);
jniThrowException(env, "java/lang/IllegalArgumentException",
sqlite3_errmsg(GET_HANDLE(env, object)));
throw_sqlite3_exception(env, GET_HANDLE(env, object));
return 0;
}
LOG_WINDOW("Bound to startPos %d", startPos);
@ -182,7 +182,8 @@ static jint native_fill_window(JNIEnv* env, jobject object, jobject javaWindow,
field_slot_t * fieldDir = window->allocRow();
if (!fieldDir) {
LOGE("Failed allocating fieldDir at startPos %d row %d", startPos, numRows);
return startPos + numRows + finish_program_and_get_row_count(statement) + 1;
gotAllRows = false;
goto return_count;
}
}
@ -207,7 +208,8 @@ static jint native_fill_window(JNIEnv* env, jobject object, jobject javaWindow,
window->freeLastRow();
LOGD("Failed allocating %u bytes for text/blob at %d,%d", size,
startPos + numRows, i);
return startPos + numRows + finish_program_and_get_row_count(statement) + 1;
gotAllRows = false;
goto return_count;
}
window->copyIn(offset, text, size);
@ -225,8 +227,9 @@ static jint native_fill_window(JNIEnv* env, jobject object, jobject javaWindow,
int64_t value = sqlite3_column_int64(statement, i);
if (!window->putLong(numRows, i, value)) {
window->freeLastRow();
LOGD("Failed allocating space for a long in column %d", i);
return startPos + numRows + finish_program_and_get_row_count(statement) + 1;
LOGE("Failed allocating space for a long in column %d", i);
gotAllRows = false;
goto return_count;
}
LOG_WINDOW("%d,%d is INTEGER 0x%016llx", startPos + numRows, i, value);
} else if (type == SQLITE_FLOAT) {
@ -234,8 +237,9 @@ static jint native_fill_window(JNIEnv* env, jobject object, jobject javaWindow,
double value = sqlite3_column_double(statement, i);
if (!window->putDouble(numRows, i, value)) {
window->freeLastRow();
LOGD("Failed allocating space for a double in column %d", i);
return startPos + numRows + finish_program_and_get_row_count(statement) + 1;
LOGE("Failed allocating space for a double in column %d", i);
gotAllRows = false;
goto return_count;
}
LOG_WINDOW("%d,%d is FLOAT %lf", startPos + numRows, i, value);
} else if (type == SQLITE_BLOB) {
@ -247,7 +251,8 @@ static jint native_fill_window(JNIEnv* env, jobject object, jobject javaWindow,
window->freeLastRow();
LOGD("Failed allocating %u bytes for blob at %d,%d", size,
startPos + numRows, i);
return startPos + numRows + finish_program_and_get_row_count(statement) + 1;
gotAllRows = false;
goto return_count;
}
window->copyIn(offset, blob, size);
@ -306,12 +311,24 @@ static jint native_fill_window(JNIEnv* env, jobject object, jobject javaWindow,
LOG_WINDOW("Resetting statement %p after fetching %d rows in %d bytes\n\n\n\n", statement,
numRows, window->size() - window->freeSpace());
// LOGI("Filled window with %d rows in %d bytes", numRows, window->size() - window->freeSpace());
LOG_WINDOW("Filled window with %d rows in %d bytes", numRows,
window->size() - window->freeSpace());
if (err == SQLITE_ROW) {
// there is more data to be returned. let the caller know by returning -1
return -1;
} else {
}
return_count:
if (startPos) {
sqlite3_reset(statement);
return startPos + numRows;
LOG_WINDOW("Not doing count(*) because startPos %d is non-zero", startPos);
return 0;
} else if (gotAllRows) {
sqlite3_reset(statement);
LOG_WINDOW("Not doing count(*) because we already know the count(*)");
return numRows;
} else {
// since startPos == 0, we need to get the count(*) of the result set
return numRows + 1 + finish_program_and_get_row_count(statement);
}
}
@ -336,7 +353,8 @@ static jstring native_column_name(JNIEnv* env, jobject object, jint columnIndex)
static JNINativeMethod sMethods[] =
{
/* name, signature, funcPtr */
{"native_fill_window", "(Landroid/database/CursorWindow;IIII)I", (void *)native_fill_window},
{"native_fill_window", "(Landroid/database/CursorWindow;IIII)I",
(void *)native_fill_window},
{"native_column_count", "()I", (void*)native_column_count},
{"native_column_name", "(I)Ljava/lang/String;", (void *)native_column_name},
};

100
core/tests/coretests/src/android/database/sqlite/SQLiteCursorTest.java
View File

@ -16,11 +16,16 @@
package android.database.sqlite;
import android.content.ContentValues;
import android.content.Context;
import android.database.Cursor;
import android.test.AndroidTestCase;
import android.test.suitebuilder.annotation.SmallTest;
import android.util.Log;
import java.io.File;
import java.util.HashSet;
import java.util.Set;
public class SQLiteCursorTest extends AndroidTestCase {
private SQLiteDatabase mDatabase;
@ -91,4 +96,99 @@ public class SQLiteCursorTest extends AndroidTestCase {
assertTrue(mDatabase.mConnectionPool.getConnectionList().contains(db));
assertTrue(db.isOpen());
}
@SmallTest
public void testFillWindow() {
// create schema
final String testTable = "testV";
mDatabase.beginTransaction();
mDatabase.execSQL("CREATE TABLE " + testTable + " (col1 int, desc text not null);");
mDatabase.setTransactionSuccessful();
mDatabase.endTransaction();
// populate the table with data
// create a big string that will almost fit a page but not quite.
// since sqlite wants to make sure each row is in a page, this string will allocate
// a new database page for each row.
StringBuilder buff = new StringBuilder();
for (int i = 0; i < 500; i++) {
buff.append(i % 10 + "");
}
ContentValues values = new ContentValues();
values.put("desc", buff.toString());
// insert more than 1MB of data in the table. this should ensure that the entire tabledata
// will need more than one CursorWindow
int N = 5000;
Set<Integer> rows = new HashSet<Integer>();
mDatabase.beginTransaction();
for (int j = 0; j < N; j++) {
values.put("col1", j);
mDatabase.insert(testTable, null, values);
rows.add(j); // store in a hashtable so we can verify the results from cursor later on
}
mDatabase.setTransactionSuccessful();
mDatabase.endTransaction();
assertEquals(N, rows.size());
Cursor c1 = mDatabase.rawQuery("select * from " + testTable, null);
assertEquals(N, c1.getCount());
c1.close();
// scroll through ALL data in the table using a cursor. should cause multiple calls to
// native_fill_window (and re-fills of the CursorWindow object)
Cursor c = mDatabase.query(testTable, new String[]{"col1", "desc"},
null, null, null, null, null);
int i = 0;
while (c.moveToNext()) {
int val = c.getInt(0);
assertTrue(rows.contains(val));
assertTrue(rows.remove(val));
}
// did I see all the rows in the table?
assertTrue(rows.isEmpty());
// change data and make sure the cursor picks up new data & count
rows = new HashSet<Integer>();
mDatabase.beginTransaction();
int M = N + 1000;
for (int j = 0; j < M; j++) {
rows.add(j);
if (j < N) {
continue;
}
values.put("col1", j);
mDatabase.insert(testTable, null, values);
}
mDatabase.setTransactionSuccessful();
mDatabase.endTransaction();
assertEquals(M, rows.size());
c.requery();
i = 0;
while (c.moveToNext()) {
int val = c.getInt(0);
assertTrue(rows.contains(val));
assertTrue(rows.remove(val));
}
// did I see all data from the modified table
assertTrue(rows.isEmpty());
// move cursor back to 1st row and scroll to about halfway in the result set
// and then delete 75% of data - and then do requery
c.moveToFirst();
int K = N / 2;
for (int p = 0; p < K && c.moveToNext(); p++) {
// nothing to do - just scrolling to about half-point in the resultset
}
mDatabase.beginTransaction();
mDatabase.delete(testTable, "col1 < ?", new String[]{ (3 * M / 4) + ""});
mDatabase.setTransactionSuccessful();
mDatabase.endTransaction();
c.requery();
assertEquals(M / 4, c.getCount());
while (c.moveToNext()) {
// just move the cursor to next row - to make sure it can go through the entire
// resultset without any problems
}
c.close();
}
}