am bae6fe24: Merge "Support streaming of compressed assets > 1 megabyte" into gingerbread

Merge commit 'bae6fe242ff9a97840ff0c5d9d87c0575d261682' into gingerbread-plus-aosp

* commit 'bae6fe242ff9a97840ff0c5d9d87c0575d261682':
  Support streaming of compressed assets > 1 megabyte

include/utils/Asset.h

@@ -61,15 +61,6 @@ public:
         ACCESS_BUFFER,
     } AccessMode;
 
-    enum {
-        /* data larger than this does not get uncompressed into a buffer */
-#ifdef HAVE_ANDROID_OS
-        UNCOMPRESS_DATA_MAX = 1 * 1024 * 1024
-#else
-        UNCOMPRESS_DATA_MAX = 2 * 1024 * 1024
-#endif
-    };
-
     /*
      * Read data from the current offset.  Returns the actual number of
      * bytes read, 0 on EOF, or -1 on error.
@@ -317,6 +308,8 @@ private:
     FileMap*    mMap;           // for memory-mapped input
     int         mFd;            // for file input
 
+    class StreamingZipInflater* mZipInflater;  // for streaming large compressed assets
+
     unsigned char*  mBuf;       // for getBuffer()
 };
 

include/utils/StreamingZipInflater.h

@@ -0,0 +1,82 @@
+/*
+ * Copyright (C) 2010 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef __LIBS_STREAMINGZIPINFLATER_H
+#define __LIBS_STREAMINGZIPINFLATER_H
+
+#include <unistd.h>
+#include <inttypes.h>
+#include <zlib.h>
+
+namespace android {
+
+class StreamingZipInflater {
+public:
+    static const size_t INPUT_CHUNK_SIZE = 64 * 1024;
+    static const size_t OUTPUT_CHUNK_SIZE = 64 * 1024;
+
+    // Flavor that pages in the compressed data from a fd
+    StreamingZipInflater(int fd, off_t compDataStart, size_t uncompSize, size_t compSize);
+
+    // Flavor that gets the compressed data from an in-memory buffer
+    StreamingZipInflater(class FileMap* dataMap, size_t uncompSize);
+
+    ~StreamingZipInflater();
+
+    // read 'count' bytes of uncompressed data from the current position.  outBuf may
+    // be NULL, in which case the data is consumed and discarded.
+    ssize_t read(void* outBuf, size_t count);
+
+    // seeking backwards requires uncompressing fom the beginning, so is very
+    // expensive.  seeking forwards only requires uncompressing from the current
+    // position to the destination.
+    off_t seekAbsolute(off_t absoluteInputPosition);
+
+private:
+    void initInflateState();
+    int readNextChunk();
+
+    // where to find the uncompressed data
+    int mFd;
+    off_t mInFileStart;         // where the compressed data lives in the file
+    class FileMap* mDataMap;
+
+    z_stream mInflateState;
+    bool mStreamNeedsInit;
+
+    // output invariants for this asset
+    uint8_t* mOutBuf;           // output buf for decompressed bytes
+    size_t mOutBufSize;         // allocated size of mOutBuf
+    size_t mOutTotalSize;       // total uncompressed size of the blob
+
+    // current output state bookkeeping
+    off_t mOutCurPosition;      // current position in total offset
+    size_t mOutLastDecoded;     // last decoded byte + 1 in mOutbuf
+    size_t mOutDeliverable;     // next undelivered byte of decoded output in mOutBuf
+
+    // input invariants
+    uint8_t* mInBuf;
+    size_t mInBufSize;          // allocated size of mInBuf;
+    size_t mInTotalSize;        // total size of compressed data for this blob
+
+    // input state bookkeeping
+    size_t mInNextChunkOffset;  // offset from start of blob at which the next input chunk lies
+    // the z_stream contains state about input block consumption
+};
+
+}
+
+#endif

libs/utils/Android.mk

@@ -33,6 +33,7 @@ commonSources:= \
 	SharedBuffer.cpp \
 	Static.cpp \
 	StopWatch.cpp \
+	StreamingZipInflater.cpp \
 	String8.cpp \
 	String16.cpp \
 	StringArray.cpp \

libs/utils/Asset.cpp

@@ -24,6 +24,7 @@
 #include <utils/Asset.h>
 #include <utils/Atomic.h>
 #include <utils/FileMap.h>
+#include <utils/StreamingZipInflater.h>
 #include <utils/ZipUtils.h>
 #include <utils/ZipFileRO.h>
 #include <utils/Log.h>
@@ -659,7 +660,7 @@ const void* _FileAsset::ensureAlignment(FileMap* map)
  */
 _CompressedAsset::_CompressedAsset(void)
     : mStart(0), mCompressedLen(0), mUncompressedLen(0), mOffset(0),
-      mMap(NULL), mFd(-1), mBuf(NULL)
+      mMap(NULL), mFd(-1), mZipInflater(NULL), mBuf(NULL)
 {
 }
 
@@ -698,6 +699,10 @@ status_t _CompressedAsset::openChunk(int fd, off_t offset,
     mFd = fd;
     assert(mBuf == NULL);
 
+    if (uncompressedLen > StreamingZipInflater::OUTPUT_CHUNK_SIZE) {
+        mZipInflater = new StreamingZipInflater(mFd, offset, uncompressedLen, compressedLen);
+    }
+
     return NO_ERROR;
 }
 
@@ -724,6 +729,9 @@ status_t _CompressedAsset::openChunk(FileMap* dataMap, int compressionMethod,
     mUncompressedLen = uncompressedLen;
     assert(mOffset == 0);
 
+    if (uncompressedLen > StreamingZipInflater::OUTPUT_CHUNK_SIZE) {
+        mZipInflater = new StreamingZipInflater(dataMap, uncompressedLen);
+    }
     return NO_ERROR;
 }
 
@@ -739,8 +747,10 @@ ssize_t _CompressedAsset::read(void* buf, size_t count)
 
     assert(mOffset >= 0 && mOffset <= mUncompressedLen);
 
-    // TODO: if mAccessMode == ACCESS_STREAMING, use zlib more cleverly
+    /* If we're relying on a streaming inflater, go through that */
-
+    if (mZipInflater) {
+        actual = mZipInflater->read(buf, count);
+    } else {
         if (mBuf == NULL) {
             if (getBuffer(false) == NULL)
                 return -1;
@@ -759,6 +769,7 @@ ssize_t _CompressedAsset::read(void* buf, size_t count)
         //printf("comp buf read\n");
         memcpy(buf, (char*)mBuf + mOffset, count);
         actual = count;
+    }
 
     mOffset += actual;
     return actual;
@@ -780,6 +791,9 @@ off_t _CompressedAsset::seek(off_t offset, int whence)
     if (newPosn == (off_t) -1)
         return newPosn;
 
+    if (mZipInflater) {
+        mZipInflater->seekAbsolute(newPosn);
+    }
     mOffset = newPosn;
     return mOffset;
 }
@@ -793,10 +807,12 @@ void _CompressedAsset::close(void)
         mMap->release();
         mMap = NULL;
     }
-    if (mBuf != NULL) {
+
     delete[] mBuf;
     mBuf = NULL;
-    }
+
+    delete mZipInflater;
+    mZipInflater = NULL;
 
     if (mFd > 0) {
         ::close(mFd);
@@ -817,12 +833,6 @@ const void* _CompressedAsset::getBuffer(bool wordAligned)
     if (mBuf != NULL)
         return mBuf;
 
-    if (mUncompressedLen > UNCOMPRESS_DATA_MAX) {
-        LOGD("Data exceeds UNCOMPRESS_DATA_MAX (%ld vs %d)\n",
-            (long) mUncompressedLen, UNCOMPRESS_DATA_MAX);
-        goto bail;
-    }
-
     /*
      * Allocate a buffer and read the file into it.
      */
@@ -853,7 +863,13 @@ const void* _CompressedAsset::getBuffer(bool wordAligned)
             goto bail;
     }
 
-    /* success! */
+    /*
+     * Success - now that we have the full asset in RAM we
+     * no longer need the streaming inflater
+     */
+    delete mZipInflater;
+    mZipInflater = NULL;
+
     mBuf = buf;
     buf = NULL;
 

libs/utils/StreamingZipInflater.cpp

@@ -0,0 +1,224 @@
+/*
+ * Copyright (C) 2010 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define LOG_TAG "szipinf"
+#include <utils/Log.h>
+
+#include <utils/FileMap.h>
+#include <utils/StreamingZipInflater.h>
+#include <string.h>
+#include <assert.h>
+
+static inline size_t min(size_t a, size_t b) { return (a < b) ? a : b; }
+
+using namespace android;
+
+/*
+ * Streaming access to compressed asset data in an open fd
+ */
+StreamingZipInflater::StreamingZipInflater(int fd, off_t compDataStart,
+        size_t uncompSize, size_t compSize) {
+    mFd = fd;
+    mDataMap = NULL;
+    mInFileStart = compDataStart;
+    mOutTotalSize = uncompSize;
+    mInTotalSize = compSize;
+
+    mInBufSize = StreamingZipInflater::INPUT_CHUNK_SIZE;
+    mInBuf = new uint8_t[mInBufSize];
+
+    mOutBufSize = StreamingZipInflater::OUTPUT_CHUNK_SIZE;
+    mOutBuf = new uint8_t[mOutBufSize];
+
+    initInflateState();
+}
+
+/*
+ * Streaming access to compressed data held in an mmapped region of memory
+ */
+StreamingZipInflater::StreamingZipInflater(FileMap* dataMap, size_t uncompSize) {
+    mFd = -1;
+    mDataMap = dataMap;
+    mOutTotalSize = uncompSize;
+    mInTotalSize = dataMap->getDataLength();
+
+    mInBuf = (uint8_t*) dataMap->getDataPtr();
+    mInBufSize = mInTotalSize;
+
+    mOutBufSize = StreamingZipInflater::OUTPUT_CHUNK_SIZE;
+    mOutBuf = new uint8_t[mOutBufSize];
+
+    initInflateState();
+}
+
+StreamingZipInflater::~StreamingZipInflater() {
+    // tear down the in-flight zip state just in case
+    ::inflateEnd(&mInflateState);
+
+    if (mDataMap == NULL) {
+        delete [] mInBuf;
+    }
+    delete [] mOutBuf;
+}
+
+void StreamingZipInflater::initInflateState() {
+    LOGD("Initializing inflate state");
+
+    memset(&mInflateState, 0, sizeof(mInflateState));
+    mInflateState.zalloc = Z_NULL;
+    mInflateState.zfree = Z_NULL;
+    mInflateState.opaque = Z_NULL;
+    mInflateState.next_in = (Bytef*)mInBuf;
+    mInflateState.next_out = (Bytef*) mOutBuf;
+    mInflateState.avail_out = mOutBufSize;
+    mInflateState.data_type = Z_UNKNOWN;
+
+    mOutLastDecoded = mOutDeliverable = mOutCurPosition = 0;
+    mInNextChunkOffset = 0;
+    mStreamNeedsInit = true;
+
+    if (mDataMap == NULL) {
+        ::lseek(mFd, mInFileStart, SEEK_SET);
+        mInflateState.avail_in = 0; // set when a chunk is read in
+    } else {
+        mInflateState.avail_in = mInBufSize;
+    }
+}
+
+/*
+ * Basic approach:
+ *
+ * 1. If we have undelivered uncompressed data, send it.  At this point
+ *    either we've satisfied the request, or we've exhausted the available
+ *    output data in mOutBuf.
+ *
+ * 2. While we haven't sent enough data to satisfy the request:
+ *    0. if the request is for more data than exists, bail.
+ *    a. if there is no input data to decode, read some into the input buffer
+ *       and readjust the z_stream input pointers
+ *    b. point the output to the start of the output buffer and decode what we can
+ *    c. deliver whatever output data we can
+ */
+ssize_t StreamingZipInflater::read(void* outBuf, size_t count) {
+    uint8_t* dest = (uint8_t*) outBuf;
+    size_t bytesRead = 0;
+    size_t toRead = min(count, size_t(mOutTotalSize - mOutCurPosition));
+    while (toRead > 0) {
+        // First, write from whatever we already have decoded and ready to go
+        size_t deliverable = min(toRead, mOutLastDecoded - mOutDeliverable);
+        if (deliverable > 0) {
+            if (outBuf != NULL) memcpy(dest, mOutBuf + mOutDeliverable, deliverable);
+            mOutDeliverable += deliverable;
+            mOutCurPosition += deliverable;
+            dest += deliverable;
+            bytesRead += deliverable;
+            toRead -= deliverable;
+        }
+
+        // need more data?  time to decode some.
+        if (toRead > 0) {
+            // if we don't have any data to decode, read some in.  If we're working
+            // from mmapped data this won't happen, because the clipping to total size
+            // will prevent reading off the end of the mapped input chunk.
+            if (mInflateState.avail_in == 0) {
+                int err = readNextChunk();
+                if (err < 0) {
+                    LOGE("Unable to access asset data: %d", err);
+                    if (!mStreamNeedsInit) {
+                        ::inflateEnd(&mInflateState);
+                        initInflateState();
+                    }
+                    return -1;
+                }
+            }
+            // we know we've drained whatever is in the out buffer now, so just
+            // start from scratch there, reading all the input we have at present.
+            mInflateState.next_out = (Bytef*) mOutBuf;
+            mInflateState.avail_out = mOutBufSize;
+
+            /*
+            LOGD("Inflating to outbuf: avail_in=%u avail_out=%u next_in=%p next_out=%p",
+                    mInflateState.avail_in, mInflateState.avail_out,
+                    mInflateState.next_in, mInflateState.next_out);
+            */
+            int result = Z_OK;
+            if (mStreamNeedsInit) {
+                LOGI("Initializing zlib to inflate");
+                result = inflateInit2(&mInflateState, -MAX_WBITS);
+                mStreamNeedsInit = false;
+            }
+            if (result == Z_OK) result = ::inflate(&mInflateState, Z_SYNC_FLUSH);
+            if (result < 0) {
+                // Whoops, inflation failed
+                LOGE("Error inflating asset: %d", result);
+                ::inflateEnd(&mInflateState);
+                initInflateState();
+                return -1;
+            } else {
+                if (result == Z_STREAM_END) {
+                    // we know we have to have reached the target size here and will
+                    // not try to read any further, so just wind things up.
+                    ::inflateEnd(&mInflateState);
+                }
+
+                // Note how much data we got, and off we go
+                mOutDeliverable = 0;
+                mOutLastDecoded = mOutBufSize - mInflateState.avail_out;
+            }
+        }
+    }
+    return bytesRead;
+}
+
+int StreamingZipInflater::readNextChunk() {
+    assert(mDataMap == NULL);
+
+    if (mInNextChunkOffset < mInTotalSize) {
+        size_t toRead = min(mInBufSize, mInTotalSize - mInNextChunkOffset);
+        if (toRead > 0) {
+            ssize_t didRead = ::read(mFd, mInBuf, toRead);
+            //LOGD("Reading input chunk, size %08x didread %08x", toRead, didRead);
+            if (didRead < 0) {
+                // TODO: error
+                LOGE("Error reading asset data");
+                return didRead;
+            } else {
+                mInNextChunkOffset += didRead;
+                mInflateState.next_in = (Bytef*) mInBuf;
+                mInflateState.avail_in = didRead;
+            }
+        }
+    }
+    return 0;
+}
+
+// seeking backwards requires uncompressing fom the beginning, so is very
+// expensive.  seeking forwards only requires uncompressing from the current
+// position to the destination.
+off_t StreamingZipInflater::seekAbsolute(off_t absoluteInputPosition) {
+    if (absoluteInputPosition < mOutCurPosition) {
+        // rewind and reprocess the data from the beginning
+        if (!mStreamNeedsInit) {
+            ::inflateEnd(&mInflateState);
+        }
+        initInflateState();
+        read(NULL, absoluteInputPosition);
+    } else if (absoluteInputPosition > mOutCurPosition) {
+        read(NULL, absoluteInputPosition - mOutCurPosition);
+    }
+    // else if the target position *is* our current position, do nothing
+    return absoluteInputPosition;
+}

tests/LargeAssetTest/Android.mk

@@ -0,0 +1,11 @@
+LOCAL_PATH:= $(call my-dir)
+include $(CLEAR_VARS)
+
+LOCAL_MODULE_TAGS := optional
+
+LOCAL_SRC_FILES := $(call all-subdir-java-files)
+
+LOCAL_PACKAGE_NAME := LargeAssetTest
+LOCAL_CERTIFICATE := platform
+
+include $(BUILD_PACKAGE)

tests/LargeAssetTest/AndroidManifest.xml

@@ -0,0 +1,28 @@
+<!-- Copyright (C) 2010 The Android Open Source Project
+
+     Licensed under the Apache License, Version 2.0 (the "License");
+     you may not use this file except in compliance with the License.
+     You may obtain a copy of the License at
+
+          http://www.apache.org/licenses/LICENSE-2.0
+
+     Unless required by applicable law or agreed to in writing, software
+     distributed under the License is distributed on an "AS IS" BASIS,
+     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+     See the License for the specific language governing permissions and
+     limitations under the License.
+-->
+<manifest xmlns:android="http://schemas.android.com/apk/res/android"
+        package="com.android.largeassettest">
+
+    <application>
+        <activity android:name="LargeAssetTest" android:label="Large Asset Test">
+            <intent-filter>
+                <action android:name="android.intent.action.MAIN" />
+                <category android:name="android.intent.category.DEFAULT" />
+                <category android:name="android.intent.category.LAUNCHER" />
+            </intent-filter>
+        </activity>
+    </application>
+
+</manifest>

tests/LargeAssetTest/res/layout/lat.xml

@@ -0,0 +1,52 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!-- Copyright (C) 2010 The Android Open Source Project
+
+     Licensed under the Apache License, Version 2.0 (the "License");
+     you may not use this file except in compliance with the License.
+     You may obtain a copy of the License at
+
+          http://www.apache.org/licenses/LICENSE-2.0
+
+     Unless required by applicable law or agreed to in writing, software
+     distributed under the License is distributed on an "AS IS" BASIS,
+     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+     See the License for the specific language governing permissions and
+     limitations under the License.
+-->
+
+<LinearLayout xmlns:android="http://schemas.android.com/apk/res/android"
+    android:layout_width="match_parent"
+    android:layout_height="match_parent"
+    android:orientation="vertical"
+    >
+
+    <TextView
+        android:layout_width="match_parent"
+        android:layout_height="wrap_content"
+        android:layout_marginTop="25dp"
+        android:textColor="#ffffffff"
+        android:text="@string/prompt"
+    />
+
+    <TextView android:id="@+id/result"
+        android:layout_width="match_parent"
+        android:layout_height="wrap_content"
+        android:layout_weight="1"
+        android:layout_marginTop="25dp"
+        android:textSize="24sp"
+        android:textColor="#ffffffff"
+    />
+
+    <Button android:id="@+id/validate"
+            android:layout_width="wrap_content"
+            android:layout_height="wrap_content"
+            android:text="@string/button_text" />
+
+    <TextView
+        android:layout_height="wrap_content"
+        android:layout_width="match_parent"
+        android:layout_weight="1"
+    />
+</LinearLayout>
+
+

tests/LargeAssetTest/res/values/strings.xml

@@ -0,0 +1,23 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!-- Copyright (C) 2010 The Android Open Source Project
+
+     Licensed under the Apache License, Version 2.0 (the "License");
+     you may not use this file except in compliance with the License.
+     You may obtain a copy of the License at
+
+          http://www.apache.org/licenses/LICENSE-2.0
+
+     Unless required by applicable law or agreed to in writing, software
+     distributed under the License is distributed on an "AS IS" BASIS,
+     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+     See the License for the specific language governing permissions and
+     limitations under the License.
+-->
+
+<resources>
+
+    <string name="prompt">Click the button below to read and validate the large binary asset.</string>
+    <string name="button_text">Validate the asset</string>
+
+</resources>
+

tests/LargeAssetTest/src/com/android/largeassettest/LargeAssetTest.java

@@ -0,0 +1,104 @@
+/*
+ * Copyright (C) 2006 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.largeassettest;
+
+import android.app.Activity;
+import android.content.Context;
+import android.content.res.AssetManager;
+import android.os.AsyncTask;
+import android.os.Bundle;
+import android.util.Log;
+import android.view.View;
+import android.widget.Button;
+import android.widget.TextView;
+
+import java.io.InputStream;
+import java.io.IOException;
+
+/**
+ * Skeleton to test large-asset handling.  The asset in question is one million
+ * four-byte integers, in ascending numeric order.
+ */
+public class LargeAssetTest extends Activity {
+    Button mValidateButton;
+    TextView mResultText;
+    Validator mValidateThread;
+
+    @Override
+    protected void onCreate(Bundle icicle) {
+        super.onCreate(icicle);
+        setContentView(R.layout.lat);
+
+        mResultText = (TextView) findViewById(R.id.result);
+        mValidateButton = (Button) findViewById(R.id.validate);
+
+        mValidateButton.setOnClickListener(mClickListener);
+    }
+
+    View.OnClickListener mClickListener = new View.OnClickListener() {
+        public void onClick(View v) {
+            mValidateButton.setEnabled(false);
+            mValidateThread = new Validator();
+            mValidateThread.execute(LargeAssetTest.this.getAssets());
+        }
+    };
+
+    /**
+     * Validation happens in a separate thread
+     */
+    class Validator extends AsyncTask<AssetManager, Integer, Boolean> {
+        static final String TAG = "Validator";
+
+        @Override
+        protected Boolean doInBackground(AssetManager... params) {
+            AssetManager am = params[0];
+            try {
+                InputStream is = am.open("million-ints", AssetManager.ACCESS_STREAMING);
+                byte[] buf = new byte[4];
+
+                for (int i = 0; i < 1000000; i++) {
+                    int num = is.read(buf, 0, 4);
+                    if (num != 4) {
+                        Log.e(TAG, "Wanted 4 bytes but read " + num);
+                        return false;
+                    }
+                    // the byte array is stored in the asset in little-endian order
+                    int value = (buf[3] << 24) + ((buf[2] & 0xFF) << 16)
+                            + ((buf[1] & 0xFF) << 8) + (buf[0] & 0xFF);
+                    if (value != i) {
+                        Log.e(TAG, "Mismatch: index " + i + " : value " + value);
+                        return false;
+                    }
+                }
+
+                is.close();
+            } catch (IOException e) {
+                Log.w(TAG, "Couldn't open asset", e);
+                return false;
+            }
+            Log.i(TAG, "Finished, reporting valid");
+            return true;
+        }
+
+        @Override
+        protected void onPostExecute(Boolean result) {
+            CharSequence text = (result) ? "Valid!" : "NOT VALID";
+            mResultText.setText(text);
+            mValidateButton.setEnabled(true);
+        }
+    }
+}