diff --git a/scripts/add-accounts b/scripts/add-accounts new file mode 100755 index 000000000..d2cddc087 --- /dev/null +++ b/scripts/add-accounts @@ -0,0 +1,131 @@ +#!/usr/bin/env python +# +# Copyright (C) 2008 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +A faux Setup Wizard. Stuffs one or two usernames + passwords into the +database on the device. +""" + +import sys +if sys.hexversion < 0x02040000: + print "This script requires python 2.4 or higher." + sys.exit(1) + +import getpass +import subprocess +import time +import sha + +DB = "/data/data/com.google.android.googleapps/databases/accounts.db" + +def RunCmd(args): + proc = subprocess.Popen(args, stdout=subprocess.PIPE) + out = proc.stdout.read() + if proc.wait(): + print + print "failed: %s" % " ".join(args) + return None + return out + +def GetProp(adb_flags, name): + args = ("adb",) + adb_flags + ("shell", "su", "root", + "/system/bin/getprop", name) + return RunCmd(args) + +def SetProp(adb_flags, name, value): + args = ("adb",) + adb_flags + ("shell", "su", "root", + "/system/bin/setprop", name, value) + return RunCmd(args) + +def DbExists(adb_flags): + args = ("adb",) + adb_flags + ("shell", "su", "root", + "/system/bin/ls", DB) + result = RunCmd(args) + if result is None: return None + return "No such file" not in result + +def main(argv): + if len(argv) == 1: + print ("usage: %s [adb flags] " + "[] " + "[]") % (argv[0],) + sys.exit(2) + + argv = argv[1:] + + gmail = None + dasher = None + while argv and "@" in argv[-1]: + addr = argv.pop() + if "@gmail.com" in addr or "@googlemail.com" in addr: + gmail = addr + else: + dasher = addr + + adb_flags = tuple(argv) + + while True: + db = DbExists(adb_flags) + if db is None: + print "failed to contact device; will retry in 3 seconds" + time.sleep(3) + continue + + if db: + print + print "GoogleLoginService has already started on this device;" + print "it's too late to use this script to add accounts." + print + print "This script only works on a freshly-wiped device (or " + print "emulator) while booting for the first time." + print + break + + hosted_account = GetProp(adb_flags, "ro.config.hosted_account").strip() + google_account = GetProp(adb_flags, "ro.config.google_account").strip() + + if dasher and hosted_account: + print + print "A dasher account is already configured on this device;" + print "can't add", hosted_account + print + dasher = None + + if gmail and google_account: + print + print "A google account is already configured on this device;" + print "can't add", google_account + print + gmail = None + + if not gmail and not dasher: break + + if dasher: + SetProp(adb_flags, "ro.config.hosted_account", dasher) + print "set hosted_account to", dasher + if gmail: + SetProp(adb_flags, "ro.config.google_account", gmail) + print "set google_account to", gmail + + break + + + + + + +if __name__ == "__main__": + main(sys.argv) diff --git a/scripts/add-accounts-sdk b/scripts/add-accounts-sdk new file mode 100755 index 000000000..bb3447fab --- /dev/null +++ b/scripts/add-accounts-sdk @@ -0,0 +1,128 @@ +#!/usr/bin/env python +# +# Copyright (C) 2008 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +A faux Setup Wizard. Stuffs one or two usernames + passwords into the +database on the device. +""" + +import sys +if sys.hexversion < 0x02040000: + print "This script requires python 2.4 or higher." + sys.exit(1) + +import getpass +import subprocess +import time +import sha + +DB = "/data/data/com.google.android.googleapps/databases/accounts.db" + +def RunCmd(args): + proc = subprocess.Popen(args, stdout=subprocess.PIPE) + out = proc.stdout.read() + if proc.wait(): + print + print "failed: %s" % " ".join(args) + return None + return out + +def GetProp(adb_flags, name): + args = ("adb",) + adb_flags + ("shell", "/system/bin/getprop", name) + return RunCmd(args) + +def SetProp(adb_flags, name, value): + args = ("adb",) + adb_flags + ("shell", "/system/bin/setprop", name, value) + return RunCmd(args) + +def DbExists(adb_flags): + args = ("adb",) + adb_flags + ("shell", "/system/bin/ls", DB) + result = RunCmd(args) + if result is None: return None + return "No such file" not in result + +def main(argv): + if len(argv) == 1: + print ("usage: %s [adb flags] " + "[] " + "[]") % (argv[0],) + sys.exit(2) + + argv = argv[1:] + + gmail = None + hosted = None + while argv and "@" in argv[-1]: + addr = argv.pop() + if "@gmail.com" in addr or "@googlemail.com" in addr: + gmail = addr + else: + hosted = addr + + adb_flags = tuple(argv) + + while True: + db = DbExists(adb_flags) + if db is None: + print "failed to contact device; will retry in 3 seconds" + time.sleep(3) + continue + + if db: + print + print "GoogleLoginService has already started on this device;" + print "it's too late to use this script to add accounts." + print + print "This script only works on a freshly-wiped device (or " + print "emulator) while booting for the first time." + print + break + + hosted_account = GetProp(adb_flags, "ro.config.hosted_account").strip() + google_account = GetProp(adb_flags, "ro.config.google_account").strip() + + if hosted and hosted_account: + print + print "A hosted account is already configured on this device;" + print "can't add", hosted_account + print + hosted = None + + if gmail and google_account: + print + print "A google account is already configured on this device;" + print "can't add", google_account + print + gmail = None + + if not gmail and not hosted: break + + if hosted: + SetProp(adb_flags, "ro.config.hosted_account", hosted) + print "set hosted_account to", hosted + if gmail: + SetProp(adb_flags, "ro.config.google_account", gmail) + print "set google_account to", gmail + + break + + + + + + +if __name__ == "__main__": + main(sys.argv) diff --git a/scripts/app_engine_server/LICENSE b/scripts/app_engine_server/LICENSE new file mode 100644 index 000000000..d64569567 --- /dev/null +++ b/scripts/app_engine_server/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/scripts/app_engine_server/app.yaml b/scripts/app_engine_server/app.yaml new file mode 100755 index 000000000..1fb50c7df --- /dev/null +++ b/scripts/app_engine_server/app.yaml @@ -0,0 +1,16 @@ +application: androidappdocs-staging +version: 1 +runtime: python +api_version: 1 + +handlers: +- url: /gae_shell/static + static_dir: gae_shell/static + expiration: 1d + +- url: /gae_shell/.* + script: /gae_shell/shell.py + login: admin + +- url: .* + script: main.py diff --git a/scripts/app_engine_server/gae_shell/README b/scripts/app_engine_server/gae_shell/README new file mode 100644 index 000000000..5b0089fef --- /dev/null +++ b/scripts/app_engine_server/gae_shell/README @@ -0,0 +1,17 @@ +An interactive, stateful AJAX shell that runs Python code on the server. + +Part of http://code.google.com/p/google-app-engine-samples/. + +May be run as a standalone app or in an existing app as an admin-only handler. +Can be used for system administration tasks, as an interactive way to try out +APIs, or as a debugging aid during development. + +The logging, os, sys, db, and users modules are imported automatically. + +Interpreter state is stored in the datastore so that variables, function +definitions, and other values in the global and local namespaces can be used +across commands. + +To use the shell in your app, copy shell.py, static/*, and templates/* into +your app's source directory. Then, copy the URL handlers from app.yaml into +your app.yaml. diff --git a/scripts/app_engine_server/gae_shell/__init__.py b/scripts/app_engine_server/gae_shell/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/scripts/app_engine_server/gae_shell/shell.py b/scripts/app_engine_server/gae_shell/shell.py new file mode 100755 index 000000000..df2fb1708 --- /dev/null +++ b/scripts/app_engine_server/gae_shell/shell.py @@ -0,0 +1,308 @@ +#!/usr/bin/python +# +# Copyright 2007 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +An interactive, stateful AJAX shell that runs Python code on the server. + +Part of http://code.google.com/p/google-app-engine-samples/. + +May be run as a standalone app or in an existing app as an admin-only handler. +Can be used for system administration tasks, as an interactive way to try out +APIs, or as a debugging aid during development. + +The logging, os, sys, db, and users modules are imported automatically. + +Interpreter state is stored in the datastore so that variables, function +definitions, and other values in the global and local namespaces can be used +across commands. + +To use the shell in your app, copy shell.py, static/*, and templates/* into +your app's source directory. Then, copy the URL handlers from app.yaml into +your app.yaml. + +TODO: unit tests! +""" + +import logging +import new +import os +import pickle +import sys +import traceback +import types +import wsgiref.handlers + +from google.appengine.api import users +from google.appengine.ext import db +from google.appengine.ext import webapp +from google.appengine.ext.webapp import template + + +# Set to True if stack traces should be shown in the browser, etc. +_DEBUG = True + +# The entity kind for shell sessions. Feel free to rename to suit your app. +_SESSION_KIND = '_Shell_Session' + +# Types that can't be pickled. +UNPICKLABLE_TYPES = ( + types.ModuleType, + types.TypeType, + types.ClassType, + types.FunctionType, + ) + +# Unpicklable statements to seed new sessions with. +INITIAL_UNPICKLABLES = [ + 'import logging', + 'import os', + 'import sys', + 'from google.appengine.ext import db', + 'from google.appengine.api import users', + ] + + +class Session(db.Model): + """A shell session. Stores the session's globals. + + Each session globals is stored in one of two places: + + If the global is picklable, it's stored in the parallel globals and + global_names list properties. (They're parallel lists to work around the + unfortunate fact that the datastore can't store dictionaries natively.) + + If the global is not picklable (e.g. modules, classes, and functions), or if + it was created by the same statement that created an unpicklable global, + it's not stored directly. Instead, the statement is stored in the + unpicklables list property. On each request, before executing the current + statement, the unpicklable statements are evaluated to recreate the + unpicklable globals. + + The unpicklable_names property stores all of the names of globals that were + added by unpicklable statements. When we pickle and store the globals after + executing a statement, we skip the ones in unpicklable_names. + + Using Text instead of string is an optimization. We don't query on any of + these properties, so they don't need to be indexed. + """ + global_names = db.ListProperty(db.Text) + globals = db.ListProperty(db.Blob) + unpicklable_names = db.ListProperty(db.Text) + unpicklables = db.ListProperty(db.Text) + + def set_global(self, name, value): + """Adds a global, or updates it if it already exists. + + Also removes the global from the list of unpicklable names. + + Args: + name: the name of the global to remove + value: any picklable value + """ + blob = db.Blob(pickle.dumps(value)) + + if name in self.global_names: + index = self.global_names.index(name) + self.globals[index] = blob + else: + self.global_names.append(db.Text(name)) + self.globals.append(blob) + + self.remove_unpicklable_name(name) + + def remove_global(self, name): + """Removes a global, if it exists. + + Args: + name: string, the name of the global to remove + """ + if name in self.global_names: + index = self.global_names.index(name) + del self.global_names[index] + del self.globals[index] + + def globals_dict(self): + """Returns a dictionary view of the globals. + """ + return dict((name, pickle.loads(val)) + for name, val in zip(self.global_names, self.globals)) + + def add_unpicklable(self, statement, names): + """Adds a statement and list of names to the unpicklables. + + Also removes the names from the globals. + + Args: + statement: string, the statement that created new unpicklable global(s). + names: list of strings; the names of the globals created by the statement. + """ + self.unpicklables.append(db.Text(statement)) + + for name in names: + self.remove_global(name) + if name not in self.unpicklable_names: + self.unpicklable_names.append(db.Text(name)) + + def remove_unpicklable_name(self, name): + """Removes a name from the list of unpicklable names, if it exists. + + Args: + name: string, the name of the unpicklable global to remove + """ + if name in self.unpicklable_names: + self.unpicklable_names.remove(name) + + +class FrontPageHandler(webapp.RequestHandler): + """Creates a new session and renders the shell.html template. + """ + + def get(self): + # set up the session. TODO: garbage collect old shell sessions + session_key = self.request.get('session') + if session_key: + session = Session.get(session_key) + else: + # create a new session + session = Session() + session.unpicklables = [db.Text(line) for line in INITIAL_UNPICKLABLES] + session_key = session.put() + + template_file = os.path.join(os.path.dirname(__file__), 'templates', + 'shell.html') + session_url = '/?session=%s' % session_key + vars = { 'server_software': os.environ['SERVER_SOFTWARE'], + 'python_version': sys.version, + 'session': str(session_key), + 'user': users.get_current_user(), + 'login_url': users.create_login_url(session_url), + 'logout_url': users.create_logout_url(session_url), + } + rendered = webapp.template.render(template_file, vars, debug=_DEBUG) + self.response.out.write(rendered) + + +class StatementHandler(webapp.RequestHandler): + """Evaluates a python statement in a given session and returns the result. + """ + + def get(self): + self.response.headers['Content-Type'] = 'text/plain' + + # extract the statement to be run + statement = self.request.get('statement') + if not statement: + return + + # the python compiler doesn't like network line endings + statement = statement.replace('\r\n', '\n') + + # add a couple newlines at the end of the statement. this makes + # single-line expressions such as 'class Foo: pass' evaluate happily. + statement += '\n\n' + + # log and compile the statement up front + try: + logging.info('Compiling and evaluating:\n%s' % statement) + compiled = compile(statement, '', 'single') + except: + self.response.out.write(traceback.format_exc()) + return + + # create a dedicated module to be used as this statement's __main__ + statement_module = new.module('__main__') + + # use this request's __builtin__, since it changes on each request. + # this is needed for import statements, among other things. + import __builtin__ + statement_module.__builtins__ = __builtin__ + + # load the session from the datastore + session = Session.get(self.request.get('session')) + + # swap in our custom module for __main__. then unpickle the session + # globals, run the statement, and re-pickle the session globals, all + # inside it. + old_main = sys.modules.get('__main__') + try: + sys.modules['__main__'] = statement_module + statement_module.__name__ = '__main__' + + # re-evaluate the unpicklables + for code in session.unpicklables: + exec code in statement_module.__dict__ + + # re-initialize the globals + for name, val in session.globals_dict().items(): + try: + statement_module.__dict__[name] = val + except: + msg = 'Dropping %s since it could not be unpickled.\n' % name + self.response.out.write(msg) + logging.warning(msg + traceback.format_exc()) + session.remove_global(name) + + # run! + old_globals = dict(statement_module.__dict__) + try: + old_stdout = sys.stdout + old_stderr = sys.stderr + try: + sys.stdout = self.response.out + sys.stderr = self.response.out + exec compiled in statement_module.__dict__ + finally: + sys.stdout = old_stdout + sys.stderr = old_stderr + except: + self.response.out.write(traceback.format_exc()) + return + + # extract the new globals that this statement added + new_globals = {} + for name, val in statement_module.__dict__.items(): + if name not in old_globals or val != old_globals[name]: + new_globals[name] = val + + if True in [isinstance(val, UNPICKLABLE_TYPES) + for val in new_globals.values()]: + # this statement added an unpicklable global. store the statement and + # the names of all of the globals it added in the unpicklables. + session.add_unpicklable(statement, new_globals.keys()) + logging.debug('Storing this statement as an unpicklable.') + + else: + # this statement didn't add any unpicklables. pickle and store the + # new globals back into the datastore. + for name, val in new_globals.items(): + if not name.startswith('__'): + session.set_global(name, val) + + finally: + sys.modules['__main__'] = old_main + + session.put() + + +def main(): + application = webapp.WSGIApplication( + [('/gae_shell/', FrontPageHandler), + ('/gae_shell/shell.do', StatementHandler)], debug=_DEBUG) + wsgiref.handlers.CGIHandler().run(application) + + +if __name__ == '__main__': + main() diff --git a/scripts/app_engine_server/gae_shell/static/shell.js b/scripts/app_engine_server/gae_shell/static/shell.js new file mode 100644 index 000000000..4aa15838a --- /dev/null +++ b/scripts/app_engine_server/gae_shell/static/shell.js @@ -0,0 +1,195 @@ +// Copyright 2007 Google Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/** + * @fileoverview + * Javascript code for the interactive AJAX shell. + * + * Part of http://code.google.com/p/google-app-engine-samples/. + * + * Includes a function (shell.runStatement) that sends the current python + * statement in the shell prompt text box to the server, and a callback + * (shell.done) that displays the results when the XmlHttpRequest returns. + * + * Also includes cross-browser code (shell.getXmlHttpRequest) to get an + * XmlHttpRequest. + */ + +/** + * Shell namespace. + * @type {Object} + */ +var shell = {} + +/** + * The shell history. history is an array of strings, ordered oldest to + * newest. historyCursor is the current history element that the user is on. + * + * The last history element is the statement that the user is currently + * typing. When a statement is run, it's frozen in the history, a new history + * element is added to the end of the array for the new statement, and + * historyCursor is updated to point to the new element. + * + * @type {Array} + */ +shell.history = ['']; + +/** + * See {shell.history} + * @type {number} + */ +shell.historyCursor = 0; + +/** + * A constant for the XmlHttpRequest 'done' state. + * @type Number + */ +shell.DONE_STATE = 4; + +/** + * A cross-browser function to get an XmlHttpRequest object. + * + * @return {XmlHttpRequest?} a new XmlHttpRequest + */ +shell.getXmlHttpRequest = function() { + if (window.XMLHttpRequest) { + return new XMLHttpRequest(); + } else if (window.ActiveXObject) { + try { + return new ActiveXObject('Msxml2.XMLHTTP'); + } catch(e) { + return new ActiveXObject('Microsoft.XMLHTTP'); + } + } + + return null; +}; + +/** + * This is the prompt textarea's onkeypress handler. Depending on the key that + * was pressed, it will run the statement, navigate the history, or update the + * current statement in the history. + * + * @param {Event} event the keypress event + * @return {Boolean} false to tell the browser not to submit the form. + */ +shell.onPromptKeyPress = function(event) { + var statement = document.getElementById('statement'); + + if (this.historyCursor == this.history.length - 1) { + // we're on the current statement. update it in the history before doing + // anything. + this.history[this.historyCursor] = statement.value; + } + + // should we pull something from the history? + if (event.ctrlKey && event.keyCode == 38 /* up arrow */) { + if (this.historyCursor > 0) { + statement.value = this.history[--this.historyCursor]; + } + return false; + } else if (event.ctrlKey && event.keyCode == 40 /* down arrow */) { + if (this.historyCursor < this.history.length - 1) { + statement.value = this.history[++this.historyCursor]; + } + return false; + } else if (!event.altKey) { + // probably changing the statement. update it in the history. + this.historyCursor = this.history.length - 1; + this.history[this.historyCursor] = statement.value; + } + + // should we submit? + var ctrlEnter = (document.getElementById('submit_key').value == 'ctrl-enter'); + if (event.keyCode == 13 /* enter */ && !event.altKey && !event.shiftKey && + event.ctrlKey == ctrlEnter) { + return this.runStatement(); + } +}; + +/** + * The XmlHttpRequest callback. If the request succeeds, it adds the command + * and its resulting output to the shell history div. + * + * @param {XmlHttpRequest} req the XmlHttpRequest we used to send the current + * statement to the server + */ +shell.done = function(req) { + if (req.readyState == this.DONE_STATE) { + var statement = document.getElementById('statement') + statement.className = 'prompt'; + + // add the command to the shell output + var output = document.getElementById('output'); + + output.value += '\n>>> ' + statement.value; + statement.value = ''; + + // add a new history element + this.history.push(''); + this.historyCursor = this.history.length - 1; + + // add the command's result + var result = req.responseText.replace(/^\s*|\s*$/g, ''); // trim whitespace + if (result != '') + output.value += '\n' + result; + + // scroll to the bottom + output.scrollTop = output.scrollHeight; + if (output.createTextRange) { + var range = output.createTextRange(); + range.collapse(false); + range.select(); + } + } +}; + +/** + * This is the form's onsubmit handler. It sends the python statement to the + * server, and registers shell.done() as the callback to run when it returns. + * + * @return {Boolean} false to tell the browser not to submit the form. + */ +shell.runStatement = function() { + var form = document.getElementById('form'); + + // build a XmlHttpRequest + var req = this.getXmlHttpRequest(); + if (!req) { + document.getElementById('ajax-status').innerHTML = + "Your browser doesn't support AJAX. :("; + return false; + } + + req.onreadystatechange = function() { shell.done(req); }; + + // build the query parameter string + var params = ''; + for (i = 0; i < form.elements.length; i++) { + var elem = form.elements[i]; + if (elem.type != 'submit' && elem.type != 'button' && elem.id != 'caret') { + var value = escape(elem.value).replace(/\+/g, '%2B'); // escape ignores + + params += '&' + elem.name + '=' + value; + } + } + + // send the request and tell the user. + document.getElementById('statement').className = 'prompt processing'; + req.open(form.method, form.action + '?' + params, true); + req.setRequestHeader('Content-type', + 'application/x-www-form-urlencoded;charset=UTF-8'); + req.send(null); + + return false; +}; diff --git a/scripts/app_engine_server/gae_shell/static/spinner.gif b/scripts/app_engine_server/gae_shell/static/spinner.gif new file mode 100644 index 000000000..3e58d6e83 Binary files /dev/null and b/scripts/app_engine_server/gae_shell/static/spinner.gif differ diff --git a/scripts/app_engine_server/gae_shell/templates/shell.html b/scripts/app_engine_server/gae_shell/templates/shell.html new file mode 100644 index 000000000..123b2009f --- /dev/null +++ b/scripts/app_engine_server/gae_shell/templates/shell.html @@ -0,0 +1,122 @@ + + + + + Interactive Shell + + + + + + +

Interactive server-side Python shell for +Google App Engine. +(source) +

+ + + +
+ + + + + + +
+ +

+ +

+{% if user %} + {{ user.nickname }} + (log out) +{% else %} + log in +{% endif %} + | Ctrl-Up/Down for history | + + +

+ + + + + + diff --git a/scripts/app_engine_server/index.yaml b/scripts/app_engine_server/index.yaml new file mode 100644 index 000000000..8e6046de9 --- /dev/null +++ b/scripts/app_engine_server/index.yaml @@ -0,0 +1,12 @@ +indexes: + +# AUTOGENERATED + +# This index.yaml is automatically updated whenever the dev_appserver +# detects that a new type of query is run. If you want to manage the +# index.yaml file manually, remove the above marker line (the line +# saying "# AUTOGENERATED"). If you want to manage some indexes +# manually, move them above the marker line. The index.yaml file is +# automatically uploaded to the admin console when you next deploy +# your application using appcfg.py. + diff --git a/scripts/app_engine_server/memcache_zipserve.py b/scripts/app_engine_server/memcache_zipserve.py new file mode 100644 index 000000000..43826b00a --- /dev/null +++ b/scripts/app_engine_server/memcache_zipserve.py @@ -0,0 +1,573 @@ +#!/usr/bin/env python +# +# Copyright 2009 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""A class to serve pages from zip files and use memcache for performance. + +This contains a class and a function to create an anonymous instance of the +class to serve HTTP GET requests. Memcache is used to increase response speed +and lower processing cycles used in serving. Credit to Guido van Rossum and +his implementation of zipserve which served as a reference as I wrote this. + + MemcachedZipHandler: Class that serves request + create_handler: method to create instance of MemcachedZipHandler +""" + +__author__ = 'jmatt@google.com (Justin Mattson)' + +import email.Utils +import logging +import mimetypes +import time +import zipfile + +from google.appengine.api import memcache +from google.appengine.ext import webapp +from google.appengine.ext.webapp import util +from time import localtime, strftime + +def create_handler(zip_files, max_age=None, public=None): + """Factory method to create a MemcachedZipHandler instance. + + Args: + zip_files: A list of file names, or a list of lists of file name, first + member of file mappings. See MemcachedZipHandler documentation for + more information about using the list of lists format + max_age: The maximum client-side cache lifetime + public: Whether this should be declared public in the client-side cache + Returns: + A MemcachedZipHandler wrapped in a pretty, anonymous bow for use with App + Engine + + Raises: + ValueError: if the zip_files argument is not a list + """ + # verify argument integrity. If the argument is passed in list format, + # convert it to list of lists format + if zip_files and type(zip_files).__name__ == 'list': + num_items = len(zip_files) + while num_items > 0: + if type(zip_files[num_items - 1]).__name__ != 'list': + zip_files[num_items - 1] = [zip_files[num_items-1]] + num_items -= 1 + else: + raise ValueError('File name arguments must be a list') + + class HandlerWrapper(MemcachedZipHandler): + """Simple wrapper for an instance of MemcachedZipHandler. + + I'm still not sure why this is needed + """ + def get(self, name): + self.zipfilenames = zip_files + self.TrueGet(name) + if max_age is not None: + MAX_AGE = max_age + if public is not None: + PUBLIC = public + + return HandlerWrapper + + +class MemcachedZipHandler(webapp.RequestHandler): + """Handles get requests for a given URL. + + Serves a GET request from a series of zip files. As files are served they are + put into memcache, which is much faster than retreiving them from the zip + source file again. It also uses considerably fewer CPU cycles. + """ + zipfile_cache = {} # class cache of source zip files + MAX_AGE = 600 # max client-side cache lifetime + PUBLIC = True # public cache setting + CACHE_PREFIX = 'cache://' # memcache key prefix for actual URLs + NEG_CACHE_PREFIX = 'noncache://' # memcache key prefix for non-existant URL + intlString = 'intl/' + validLangs = ['en', 'de', 'es', 'fr','it','ja','zh-CN','zh-TW'] + + def TrueGet(self, reqUri): + """The top-level entry point to serving requests. + + Called 'True' get because it does the work when called from the wrapper + class' get method. Some logic is applied to the request to serve files + from an intl//... directory or fall through to the default language. + + Args: + name: URL requested + + Returns: + None + """ + langName = 'en' + resetLangCookie = False + urlLangName = None + retry = False + isValidIntl = False + + # Try to retrieve the user's lang pref from the cookie. If there is no + # lang pref cookie in the request, add set-cookie to the response with the + # default value of 'en'. + try: + langName = self.request.cookies['android_developer_pref_lang'] + except KeyError: + resetLangCookie = True + #logging.info('==========================EXCEPTION: NO LANG COOKIE FOUND, USING [%s]', langName) + logging.info('==========================REQ INIT name [%s] langName [%s]', reqUri, langName) + + # Preprocess the req url. If it references a directory or the domain itself, + # append '/index.html' to the url and 302 redirect. Otherwise, continue + # processing the request below. + name = self.PreprocessUrl(reqUri, langName) + if name: + # Do some prep for handling intl requests. Parse the url and validate + # the intl/lang substring, extract the url lang code (urlLangName) and the + # the uri that follows the intl/lang substring(contentUri) + sections = name.split("/", 2) + contentUri = 0 + isIntl = len(sections) > 1 and (sections[0] == "intl") + if isIntl: + isValidIntl = sections[1] in self.validLangs + if isValidIntl: + urlLangName = sections[1] + contentUri = sections[2] + if (langName != urlLangName): + # if the lang code in the request is different from that in + # the cookie, reset the cookie to the url lang value. + langName = urlLangName + resetLangCookie = True + #logging.info('INTL PREP resetting langName to urlLangName [%s]', langName) + #else: + # logging.info('INTL PREP no need to reset langName') + + # Send for processing + if self.isCleanUrl(name, langName, isValidIntl): + # handle a 'clean' request. + # Try to form a response using the actual request url. + if not self.CreateResponse(name, langName, isValidIntl, resetLangCookie): + # If CreateResponse returns False, there was no such document + # in the intl/lang tree. Before going to 404, see if there is an + # English-language version of the doc in the default + # default tree and return it, else go to 404. + self.CreateResponse(contentUri, langName, False, resetLangCookie) + + elif isIntl: + # handle the case where we need to pass through an invalid intl req + # for processing (so as to get 404 as appropriate). This is needed + # because intl urls are passed through clean and retried in English, + # if necessary. + logging.info(' Handling an invalid intl request...') + self.CreateResponse(name, langName, isValidIntl, resetLangCookie) + + else: + # handle the case where we have a non-clean url (usually a non-intl + # url) that we need to interpret in the context of any lang pref + # that is set. Prepend an intl/lang string to the request url and + # send it as a 302 redirect. After the redirect, the subsequent + # request will be handled as a clean url. + self.RedirToIntl(name, self.intlString, langName) + + def isCleanUrl(self, name, langName, isValidIntl): + """Determine whether to pass an incoming url straight to processing. + + Args: + name: The incoming URL + + Returns: + boolean: Whether the URL should be sent straight to processing + """ + if (langName == 'en') or isValidIntl or not ('.html' in name) or (not isValidIntl and not langName): + return True + + def PreprocessUrl(self, name, langName): + """Any preprocessing work on the URL when it comes in. + + Put any work related to interpretting the incoming URL here. For example, + this is used to redirect requests for a directory to the index.html file + in that directory. Subclasses should override this method to do different + preprocessing. + + Args: + name: The incoming URL + + Returns: + False if the request was redirected to '/index.html', or + The processed URL, otherwise + """ + # determine if this is a request for a directory + final_path_segment = name + final_slash_offset = name.rfind('/') + if final_slash_offset != len(name) - 1: + final_path_segment = name[final_slash_offset + 1:] + if final_path_segment.find('.') == -1: + name = ''.join([name, '/']) + + # if this is a directory or the domain itself, redirect to /index.html + if not name or (name[len(name) - 1:] == '/'): + uri = ''.join(['/', name, 'index.html']) + logging.info('--->PREPROCESSING REDIRECT [%s] to [%s] with langName [%s]', name, uri, langName) + self.redirect(uri, False) + return False + else: + return name + + def RedirToIntl(self, name, intlString, langName): + """Redirect an incoming request to the appropriate intl uri. + + Builds the intl/lang string from a base (en) string + and redirects (302) the request to look for a version + of the file in the language that matches the client- + supplied cookie value. + + Args: + name: The incoming, preprocessed URL + + Returns: + The lang-specific URL + """ + builtIntlLangUri = ''.join([intlString, langName, '/', name, '?', self.request.query_string]) + uri = ''.join(['/', builtIntlLangUri]) + logging.info('-->>REDIRECTING %s to %s', name, uri) + self.redirect(uri, False) + return uri + + def CreateResponse(self, name, langName, isValidIntl, resetLangCookie): + """Process the url and form a response, if appropriate. + + Attempts to retrieve the requested file (name) from cache, + negative cache, or store (zip) and form the response. + For intl requests that are not found (in the localized tree), + returns False rather than forming a response, so that + the request can be retried with the base url (this is the + fallthrough to default language). + + For requests that are found, forms the headers and + adds the content to the response entity. If the request was + for an intl (localized) url, also resets the language cookie + to the language specified in the url if needed, to ensure that + the client language and response data remain harmonious. + + Args: + name: The incoming, preprocessed URL + langName: The language id. Used as necessary to reset the + language cookie in the response. + isValidIntl: If present, indicates whether the request is + for a language-specific url + resetLangCookie: Whether the response should reset the + language cookie to 'langName' + + Returns: + True: A response was successfully created for the request + False: No response was created. + """ + # see if we have the page in the memcache + logging.info('PROCESSING %s langName [%s] isValidIntl [%s] resetLang [%s]', + name, langName, isValidIntl, resetLangCookie) + resp_data = self.GetFromCache(name) + if resp_data is None: + logging.info(' Cache miss for %s', name) + resp_data = self.GetFromNegativeCache(name) + if resp_data is None: + resp_data = self.GetFromStore(name) + + # IF we have the file, put it in the memcache + # ELSE put it in the negative cache + if resp_data is not None: + self.StoreOrUpdateInCache(name, resp_data) + elif isValidIntl: + # couldn't find the intl doc. Try to fall through to English. + #logging.info(' Retrying with base uri...') + return False + else: + logging.info(' Adding %s to negative cache, serving 404', name) + self.StoreInNegativeCache(name) + self.Write404Error() + return True + else: + # found it in negative cache + self.Write404Error() + return True + + # found content from cache or store + logging.info('FOUND CLEAN') + if resetLangCookie: + logging.info(' Resetting android_developer_pref_lang cookie to [%s]', + langName) + expireDate = time.mktime(localtime()) + 60 * 60 * 24 * 365 * 10 + self.response.headers.add_header('Set-Cookie', + 'android_developer_pref_lang=%s; path=/; expires=%s' % + (langName, strftime("%a, %d %b %Y %H:%M:%S", localtime(expireDate)))) + mustRevalidate = False + if ('.html' in name): + # revalidate html files -- workaround for cache inconsistencies for + # negotiated responses + mustRevalidate = True + logging.info(' Adding [Vary: Cookie] to response...') + self.response.headers.add_header('Vary', 'Cookie') + content_type, encoding = mimetypes.guess_type(name) + if content_type: + self.response.headers['Content-Type'] = content_type + self.SetCachingHeaders(mustRevalidate) + self.response.out.write(resp_data) + elif (name == 'favicon.ico'): + self.response.headers['Content-Type'] = 'image/x-icon' + self.SetCachingHeaders(mustRevalidate) + self.response.out.write(resp_data) + elif name.endswith('.psd'): + self.response.headers['Content-Type'] = 'application/octet-stream' + self.SetCachingHeaders(mustRevalidate) + self.response.out.write(resp_data) + return True + + def GetFromStore(self, file_path): + """Retrieve file from zip files. + + Get the file from the source, it must not have been in the memcache. If + possible, we'll use the zip file index to quickly locate where the file + should be found. (See MapToFileArchive documentation for assumptions about + file ordering.) If we don't have an index or don't find the file where the + index says we should, look through all the zip files to find it. + + Args: + file_path: the file that we're looking for + + Returns: + The contents of the requested file + """ + resp_data = None + file_itr = iter(self.zipfilenames) + + # check the index, if we have one, to see what archive the file is in + archive_name = self.MapFileToArchive(file_path) + if not archive_name: + archive_name = file_itr.next()[0] + + while resp_data is None and archive_name: + zip_archive = self.LoadZipFile(archive_name) + if zip_archive: + + # we expect some lookups will fail, and that's okay, 404s will deal + # with that + try: + resp_data = zip_archive.read(file_path) + except (KeyError, RuntimeError), err: + # no op + x = False + if resp_data is not None: + logging.info('%s read from %s', file_path, archive_name) + + try: + archive_name = file_itr.next()[0] + except (StopIteration), err: + archive_name = False + + return resp_data + + def LoadZipFile(self, zipfilename): + """Convenience method to load zip file. + + Just a convenience method to load the zip file from the data store. This is + useful if we ever want to change data stores and also as a means of + dependency injection for testing. This method will look at our file cache + first, and then load and cache the file if there's a cache miss + + Args: + zipfilename: the name of the zip file to load + + Returns: + The zip file requested, or None if there is an I/O error + """ + zip_archive = None + zip_archive = self.zipfile_cache.get(zipfilename) + if zip_archive is None: + try: + zip_archive = zipfile.ZipFile(zipfilename) + self.zipfile_cache[zipfilename] = zip_archive + except (IOError, RuntimeError), err: + logging.error('Can\'t open zipfile %s, cause: %s' % (zipfilename, + err)) + return zip_archive + + def MapFileToArchive(self, file_path): + """Given a file name, determine what archive it should be in. + + This method makes two critical assumptions. + (1) The zip files passed as an argument to the handler, if concatenated + in that same order, would result in a total ordering + of all the files. See (2) for ordering type. + (2) Upper case letters before lower case letters. The traversal of a + directory tree is depth first. A parent directory's files are added + before the files of any child directories + + Args: + file_path: the file to be mapped to an archive + + Returns: + The name of the archive where we expect the file to be + """ + num_archives = len(self.zipfilenames) + while num_archives > 0: + target = self.zipfilenames[num_archives - 1] + if len(target) > 1: + if self.CompareFilenames(target[1], file_path) >= 0: + return target[0] + num_archives -= 1 + + return None + + def CompareFilenames(self, file1, file2): + """Determines whether file1 is lexigraphically 'before' file2. + + WARNING: This method assumes that paths are output in a depth-first, + with parent directories' files stored before childs' + + We say that file1 is lexigraphically before file2 if the last non-matching + path segment of file1 is alphabetically before file2. + + Args: + file1: the first file path + file2: the second file path + + Returns: + A positive number if file1 is before file2 + A negative number if file2 is before file1 + 0 if filenames are the same + """ + f1_segments = file1.split('/') + f2_segments = file2.split('/') + + segment_ptr = 0 + while (segment_ptr < len(f1_segments) and + segment_ptr < len(f2_segments) and + f1_segments[segment_ptr] == f2_segments[segment_ptr]): + segment_ptr += 1 + + if len(f1_segments) == len(f2_segments): + + # we fell off the end, the paths much be the same + if segment_ptr == len(f1_segments): + return 0 + + # we didn't fall of the end, compare the segments where they differ + if f1_segments[segment_ptr] < f2_segments[segment_ptr]: + return 1 + elif f1_segments[segment_ptr] > f2_segments[segment_ptr]: + return -1 + else: + return 0 + + # the number of segments differs, we either mismatched comparing + # directories, or comparing a file to a directory + else: + + # IF we were looking at the last segment of one of the paths, + # the one with fewer segments is first because files come before + # directories + # ELSE we just need to compare directory names + if (segment_ptr + 1 == len(f1_segments) or + segment_ptr + 1 == len(f2_segments)): + return len(f2_segments) - len(f1_segments) + else: + if f1_segments[segment_ptr] < f2_segments[segment_ptr]: + return 1 + elif f1_segments[segment_ptr] > f2_segments[segment_ptr]: + return -1 + else: + return 0 + + def SetCachingHeaders(self, revalidate): + """Set caching headers for the request.""" + max_age = self.MAX_AGE + #self.response.headers['Expires'] = email.Utils.formatdate( + # time.time() + max_age, usegmt=True) + cache_control = [] + if self.PUBLIC: + cache_control.append('public') + cache_control.append('max-age=%d' % max_age) + if revalidate: + cache_control.append('must-revalidate') + self.response.headers['Cache-Control'] = ', '.join(cache_control) + + def GetFromCache(self, filename): + """Get file from memcache, if available. + + Args: + filename: The URL of the file to return + + Returns: + The content of the file + """ + return memcache.get('%s%s' % (self.CACHE_PREFIX, filename)) + + def StoreOrUpdateInCache(self, filename, data): + """Store data in the cache. + + Store a piece of data in the memcache. Memcache has a maximum item size of + 1*10^6 bytes. If the data is too large, fail, but log the failure. Future + work will consider compressing the data before storing or chunking it + + Args: + filename: the name of the file to store + data: the data of the file + + Returns: + None + """ + try: + if not memcache.add('%s%s' % (self.CACHE_PREFIX, filename), data): + memcache.replace('%s%s' % (self.CACHE_PREFIX, filename), data) + except (ValueError), err: + logging.warning('Data size too large to cache\n%s' % err) + + def Write404Error(self): + """Ouptut a simple 404 response.""" + self.error(404) + self.response.out.write( + ''.join(['404: Not Found', + '

Error 404


', + 'File not found'])) + + def StoreInNegativeCache(self, filename): + """If a non-existant URL is accessed, cache this result as well. + + Future work should consider setting a maximum negative cache size to + prevent it from from negatively impacting the real cache. + + Args: + filename: URL to add ot negative cache + + Returns: + None + """ + memcache.add('%s%s' % (self.NEG_CACHE_PREFIX, filename), -1) + + def GetFromNegativeCache(self, filename): + """Retrieve from negative cache. + + Args: + filename: URL to retreive + + Returns: + The file contents if present in the negative cache. + """ + return memcache.get('%s%s' % (self.NEG_CACHE_PREFIX, filename)) + +def main(): + application = webapp.WSGIApplication([('/([^/]+)/(.*)', + MemcachedZipHandler)]) + util.run_wsgi_app(application) + + +if __name__ == '__main__': + main() diff --git a/scripts/combine_sdks.sh b/scripts/combine_sdks.sh new file mode 100755 index 000000000..ebaa1c6ee --- /dev/null +++ b/scripts/combine_sdks.sh @@ -0,0 +1,105 @@ +#!/bin/bash + +function replace() +{ + echo replacing $1 + rm $V -rf "$UNZIPPED_BASE_DIR"/$1 + cp $V -rf "$UNZIPPED_IMAGE_DIR"/$1 "$UNZIPPED_BASE_DIR"/$1 +} + +V="" +Q="-q" +if [ "$1" == "-v" ]; then + V="-v" + Q="" + shift +fi + +NOZIP="" +if [ "$1" == "-nozip" ]; then + NOZIP="1" + shift +fi + +BASE="$1" +IMAGES="$2" +OUTPUT="$3" + +if [[ -z "$BASE" || -z "$IMAGES" || -z "$OUTPUT" ]] ; then + echo "usage: combine_sdks.sh [-v] [-nozip] BASE IMAGES OUTPUT" + echo + echo " BASE and IMAGES should be sdk zip files. The system image files," + echo " emulator and other runtime files will be copied from IMAGES and" + echo " everything else will be copied from BASE. All of this will be" + echo " bundled into OUTPUT and zipped up again (unless -nozip is specified)." + echo + exit 1 +fi + +TMP=$(mktemp -d) + +TMP_ZIP=tmp.zip + +# determine executable extension +case `uname -s` in + *_NT-*) # for Windows + EXE=.exe + ;; + *) + EXE= + ;; +esac + +BASE_DIR="$TMP"/base +IMAGES_DIR="$TMP"/images +OUTPUT_TMP_ZIP="$BASE_DIR/$TMP_ZIP" + +unzip $Q "$BASE" -d "$BASE_DIR" +unzip $Q "$IMAGES" -d "$IMAGES_DIR" + +UNZIPPED_BASE_DIR=$(echo "$BASE_DIR"/*) +UNZIPPED_IMAGE_DIR=$(echo "$IMAGES_DIR"/*) + +# +# The commands to copy over the files that we want +# + +# replace tools/emulator # at this time we do not want the exe from SDK1.x +replace tools/lib/images +replace tools/lib/res +replace tools/lib/fonts +replace tools/lib/layoutlib.jar +replace docs +replace android.jar + +for i in widgets categories broadcast_actions service_actions; do + replace tools/lib/$i.txt +done + +if [ -d "$UNZIPPED_BASE_DIR"/usb_driver ]; then + replace usb_driver +fi + +# +# end +# + +if [ -z "$NOZIP" ]; then + pushd "$BASE_DIR" &> /dev/null + # rename the directory to the leaf minus the .zip of OUTPUT + LEAF=$(echo "$OUTPUT" | sed -e "s:.*\.zip/::" | sed -e "s:.zip$::") + mv * "$LEAF" + # zip it + zip $V -qr "$TMP_ZIP" "$LEAF" + popd &> /dev/null + + cp $V "$OUTPUT_TMP_ZIP" "$OUTPUT" + echo "Combined SDK available at $OUTPUT" +else + OUT_DIR="${OUTPUT//.zip/}" + mv $V "$BASE_DIR"/* "$OUT_DIR" + echo "Unzipped combined SDK available at $OUT_DIR" +fi + +rm $V -rf "$TMP" + diff --git a/scripts/divide_and_compress.py b/scripts/divide_and_compress.py new file mode 100755 index 000000000..2bcb0ab67 --- /dev/null +++ b/scripts/divide_and_compress.py @@ -0,0 +1,366 @@ +#!/usr/bin/python2.4 +# +# Copyright (C) 2008 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Module to compress directories in to series of zip files. + +This module will take a directory and compress all its contents, including +child directories into a series of zip files named N.zip where 'N' ranges from +0 to infinity. The zip files will all be below a certain specified maximum +threshold. + +The directory is compressed with a depth first traversal, each directory's +file contents being compressed as it is visisted, before the compression of any +child directory's contents. In this way the files within an archive are ordered +and the archives themselves are ordered. + +The class also constructs a 'main.py' file intended for use with Google App +Engine with a custom App Engine program not currently distributed with this +code base. The custom App Engine runtime can leverage the index files written +out by this class to more quickly locate which zip file to serve a given URL +from. +""" + +__author__ = 'jmatt@google.com (Justin Mattson)' + +import optparse +import os +import stat +import sys +import zipfile +import divide_and_compress_constants + + +def CreateOptionsParser(): + """Creates the parser for command line arguments. + + Returns: + A configured optparse.OptionParser object. + """ + rtn = optparse.OptionParser() + rtn.add_option('-s', '--sourcefiles', dest='sourcefiles', default=None, + help='The directory containing the files to compress') + rtn.add_option('-d', '--destination', dest='destination', default=None, + help=('Where to put the archive files, this should not be' + ' a child of where the source files exist.')) + rtn.add_option('-f', '--filesize', dest='filesize', default='1M', + help=('Maximum size of archive files. A number followed by ' + 'a magnitude indicator either "B", "K", "M", or "G". ' + 'Examples:\n 1000000B == one million BYTES\n' + ' 1.2M == one point two MEGABYTES\n' + ' 1M == 1048576 BYTES')) + rtn.add_option('-n', '--nocompress', action='store_false', dest='compress', + default=True, + help=('Whether the archive files should be compressed, or ' + 'just a concatenation of the source files')) + return rtn + + +def VerifyArguments(options, parser): + """Runs simple checks on correctness of commandline arguments. + + Args: + options: The command line options passed. + parser: The parser object used to parse the command string. + """ + try: + if options.sourcefiles is None or options.destination is None: + parser.print_help() + sys.exit(-1) + except AttributeError: + parser.print_help() + sys.exit(-1) + + +def ParseSize(size_str): + """Parse the file size argument from a string to a number of bytes. + + Args: + size_str: The string representation of the file size. + + Returns: + The file size in bytes. + + Raises: + ValueError: Raises an error if the numeric or qualifier portions of the + file size argument is invalid. + """ + if len(size_str) < 2: + raise ValueError(('filesize argument not understood, please include' + ' a numeric value and magnitude indicator')) + magnitude = size_str[-1] + if not magnitude in ('B', 'K', 'M', 'G'): + raise ValueError(('filesize magnitude indicator not valid, must be "B",' + '"K","M", or "G"')) + numeral = float(size_str[:-1]) + if magnitude == 'K': + numeral *= 1024 + elif magnitude == 'M': + numeral *= 1048576 + elif magnitude == 'G': + numeral *= 1073741824 + return int(numeral) + + +class DirectoryZipper(object): + """Class to compress a directory and all its sub-directories.""" + + def __init__(self, output_path, base_dir, archive_size, enable_compression): + """DirectoryZipper constructor. + + Args: + output_path: A string, the path to write the archives and index file to. + base_dir: A string, the directory to compress. + archive_size: An number, the maximum size, in bytes, of a single + archive file. + enable_compression: A boolean, whether or not compression should be + enabled, if disabled, the files will be written into an uncompresed + zip. + """ + self.output_dir = output_path + self.current_archive = '0.zip' + self.base_path = base_dir + self.max_size = archive_size + self.compress = enable_compression + + # Set index_fp to None, because we don't know what it will be yet. + self.index_fp = None + + def StartCompress(self): + """Start compress of the directory. + + This will start the compression process and write the archives to the + specified output directory. It will also produce an 'index.txt' file in the + output directory that maps from file to archive. + """ + self.index_fp = open(os.path.join(self.output_dir, 'main.py'), 'w') + self.index_fp.write(divide_and_compress_constants.file_preamble) + os.path.walk(self.base_path, self.CompressDirectory, 1) + self.index_fp.write(divide_and_compress_constants.file_endpiece) + self.index_fp.close() + + def RemoveLastFile(self, archive_path=None): + """Removes the last item in the archive. + + This removes the last item in the archive by reading the items out of the + archive, adding them to a new archive, deleting the old archive, and + moving the new archive to the location of the old archive. + + Args: + archive_path: Path to the archive to modify. This archive should not be + open elsewhere, since it will need to be deleted. + + Returns: + A new ZipFile object that points to the modified archive file. + """ + if archive_path is None: + archive_path = os.path.join(self.output_dir, self.current_archive) + + # Move the old file and create a new one at its old location. + root, ext = os.path.splitext(archive_path) + old_archive = ''.join([root, '-old', ext]) + os.rename(archive_path, old_archive) + old_fp = self.OpenZipFileAtPath(old_archive, mode='r') + + # By default, store uncompressed. + compress_bit = zipfile.ZIP_STORED + if self.compress: + compress_bit = zipfile.ZIP_DEFLATED + new_fp = self.OpenZipFileAtPath(archive_path, + mode='w', + compress=compress_bit) + + # Read the old archive in a new archive, except the last one. + for zip_member in old_fp.infolist()[:-1]: + new_fp.writestr(zip_member, old_fp.read(zip_member.filename)) + + # Close files and delete the old one. + old_fp.close() + new_fp.close() + os.unlink(old_archive) + + def OpenZipFileAtPath(self, path, mode=None, compress=zipfile.ZIP_DEFLATED): + """This method is mainly for testing purposes, eg dependency injection.""" + if mode is None: + if os.path.exists(path): + mode = 'a' + else: + mode = 'w' + + if mode == 'r': + return zipfile.ZipFile(path, mode) + else: + return zipfile.ZipFile(path, mode, compress) + + def CompressDirectory(self, unused_id, dir_path, dir_contents): + """Method to compress the given directory. + + This method compresses the directory 'dir_path'. It will add to an existing + zip file that still has space and create new ones as necessary to keep zip + file sizes under the maximum specified size. This also writes out the + mapping of files to archives to the self.index_fp file descriptor + + Args: + unused_id: A numeric identifier passed by the os.path.walk method, this + is not used by this method. + dir_path: A string, the path to the directory to compress. + dir_contents: A list of directory contents to be compressed. + """ + # Construct the queue of files to be added that this method will use + # it seems that dir_contents is given in reverse alphabetical order, + # so put them in alphabetical order by inserting to front of the list. + dir_contents.sort() + zip_queue = [] + for filename in dir_contents: + zip_queue.append(os.path.join(dir_path, filename)) + compress_bit = zipfile.ZIP_DEFLATED + if not self.compress: + compress_bit = zipfile.ZIP_STORED + + # Zip all files in this directory, adding to existing archives and creating + # as necessary. + while zip_queue: + target_file = zip_queue[0] + if os.path.isfile(target_file): + self.AddFileToArchive(target_file, compress_bit) + + # See if adding the new file made our archive too large. + if not self.ArchiveIsValid(): + + # IF fixing fails, the last added file was to large, skip it + # ELSE the current archive filled normally, make a new one and try + # adding the file again. + if not self.FixArchive('SIZE'): + zip_queue.pop(0) + else: + self.current_archive = '%i.zip' % ( + int(self.current_archive[ + 0:self.current_archive.rfind('.zip')]) + 1) + else: + + # Write an index record if necessary. + self.WriteIndexRecord() + zip_queue.pop(0) + else: + zip_queue.pop(0) + + def WriteIndexRecord(self): + """Write an index record to the index file. + + Only write an index record if this is the first file to go into archive + + Returns: + True if an archive record is written, False if it isn't. + """ + archive = self.OpenZipFileAtPath( + os.path.join(self.output_dir, self.current_archive), 'r') + archive_index = archive.infolist() + if len(archive_index) == 1: + self.index_fp.write( + '[\'%s\', \'%s\'],\n' % (self.current_archive, + archive_index[0].filename)) + archive.close() + return True + else: + archive.close() + return False + + def FixArchive(self, problem): + """Make the archive compliant. + + Args: + problem: An enum, the reason the archive is invalid. + + Returns: + Whether the file(s) removed to fix the archive could conceivably be + in an archive, but for some reason can't be added to this one. + """ + archive_path = os.path.join(self.output_dir, self.current_archive) + return_value = None + + if problem == 'SIZE': + archive_obj = self.OpenZipFileAtPath(archive_path, mode='r') + num_archive_files = len(archive_obj.infolist()) + + # IF there is a single file, that means its too large to compress, + # delete the created archive + # ELSE do normal finalization. + if num_archive_files == 1: + print ('WARNING: %s%s is too large to store.' % ( + self.base_path, archive_obj.infolist()[0].filename)) + archive_obj.close() + os.unlink(archive_path) + return_value = False + else: + archive_obj.close() + self.RemoveLastFile( + os.path.join(self.output_dir, self.current_archive)) + print 'Final archive size for %s is %i' % ( + self.current_archive, os.path.getsize(archive_path)) + return_value = True + return return_value + + def AddFileToArchive(self, filepath, compress_bit): + """Add the file at filepath to the current archive. + + Args: + filepath: A string, the path of the file to add. + compress_bit: A boolean, whether or not this file should be compressed + when added. + + Returns: + True if the file could be added (typically because this is a file) or + False if it couldn't be added (typically because its a directory). + """ + curr_archive_path = os.path.join(self.output_dir, self.current_archive) + if os.path.isfile(filepath) and not os.path.islink(filepath): + if os.path.getsize(filepath) > 1048576: + print 'Warning: %s is potentially too large to serve on GAE' % filepath + archive = self.OpenZipFileAtPath(curr_archive_path, + compress=compress_bit) + # Add the file to the archive. + archive.write(filepath, filepath[len(self.base_path):]) + archive.close() + return True + else: + return False + + def ArchiveIsValid(self): + """Check whether the archive is valid. + + Currently this only checks whether the archive is under the required size. + The thought is that eventually this will do additional validation + + Returns: + True if the archive is valid, False if its not. + """ + archive_path = os.path.join(self.output_dir, self.current_archive) + return os.path.getsize(archive_path) <= self.max_size + + +def main(argv): + parser = CreateOptionsParser() + (options, unused_args) = parser.parse_args(args=argv[1:]) + VerifyArguments(options, parser) + zipper = DirectoryZipper(options.destination, + options.sourcefiles, + ParseSize(options.filesize), + options.compress) + zipper.StartCompress() + + +if __name__ == '__main__': + main(sys.argv) diff --git a/scripts/divide_and_compress_constants.py b/scripts/divide_and_compress_constants.py new file mode 100644 index 000000000..15162b712 --- /dev/null +++ b/scripts/divide_and_compress_constants.py @@ -0,0 +1,58 @@ +#!/usr/bin/python2.4 +# +# Copyright (C) 2008 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Constants for the divide_and_compress script and DirectoryZipper class.""" + +__author__ = 'jmatt@google.com (Justin Mattson)' + +file_preamble = """#!/usr/bin/env python +# +# Copyright 2008 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an \"AS IS\" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import wsgiref.handlers\n' +from google.appengine.ext import zipserve\n' +from google.appengine.ext import webapp\n' +import memcache_zipserve\n\n\n' +class MainHandler(webapp.RequestHandler): + + def get(self): + self.response.out.write('Hello world!') + +def main(): + application = webapp.WSGIApplication(['/(.*)', + memcache_zipserve.create_handler([""" + +file_endpiece = """])), +], +debug=False) + wsgiref.handlers.CGIHandler().run(application) + +if __name__ == __main__: + main()""" diff --git a/scripts/divide_and_compress_test.py b/scripts/divide_and_compress_test.py new file mode 100755 index 000000000..426449af6 --- /dev/null +++ b/scripts/divide_and_compress_test.py @@ -0,0 +1,489 @@ +#!/usr/bin/python2.4 +# +# Copyright (C) 2008 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Tests for divide_and_compress.py. + +TODO(jmatt): Add tests for module methods. +""" + +__author__ = 'jmatt@google.com (Justin Mattson)' + +import os +import stat +import unittest +import zipfile + +import divide_and_compress +import mox + + +class BagOfParts(object): + """Just a generic class that I can use to assign random attributes to.""" + + def NoOp(self): + x = 1 + + +class ValidAndRemoveTests(unittest.TestCase): + """Test the ArchiveIsValid and RemoveLastFile methods.""" + + def setUp(self): + """Prepare the test. + + Construct some mock objects for use with the tests. + """ + self.my_mox = mox.Mox() + file1 = BagOfParts() + file1.filename = 'file1.txt' + file1.contents = 'This is a test file' + file2 = BagOfParts() + file2.filename = 'file2.txt' + file2.contents = ('akdjfk;djsf;kljdslkfjslkdfjlsfjkdvn;kn;2389rtu4i' + 'tn;ghf8:89H*hp748FJw80fu9WJFpwf39pujens;fihkhjfk' + 'sdjfljkgsc n;iself') + self.files = {'file1': file1, 'file2': file2} + + def tearDown(self): + """Remove any stubs we've created.""" + self.my_mox.UnsetStubs() + + def testArchiveIsValid(self): + """Test the DirectoryZipper.ArchiveIsValid method. + + Run two tests, one that we expect to pass and one that we expect to fail + """ + test_file_size = 1056730 + self.my_mox.StubOutWithMock(os, 'stat') + os.stat('/foo/0.zip').AndReturn([test_file_size]) + self.my_mox.StubOutWithMock(stat, 'ST_SIZE') + stat.ST_SIZE = 0 + os.stat('/baz/0.zip').AndReturn([test_file_size]) + mox.Replay(os.stat) + test_target = divide_and_compress.DirectoryZipper('/foo/', 'bar', + test_file_size - 1, True) + + self.assertEqual(False, test_target.ArchiveIsValid(), + msg=('ERROR: Test failed, ArchiveIsValid should have ' + 'returned false, but returned true')) + + test_target = divide_and_compress.DirectoryZipper('/baz/', 'bar', + test_file_size + 1, True) + self.assertEqual(True, test_target.ArchiveIsValid(), + msg=('ERROR: Test failed, ArchiveIsValid should have' + ' returned true, but returned false')) + + def testRemoveLastFile(self): + """Test DirectoryZipper.RemoveLastFile method. + + Construct a ZipInfo mock object with two records, verify that write is + only called once on the new ZipFile object. + """ + source = self.CreateZipSource() + dest = self.CreateZipDestination() + source_path = ''.join([os.getcwd(), '/0-old.zip']) + dest_path = ''.join([os.getcwd(), '/0.zip']) + test_target = divide_and_compress.DirectoryZipper( + ''.join([os.getcwd(), '/']), 'dummy', 1024*1024, True) + self.my_mox.StubOutWithMock(test_target, 'OpenZipFileAtPath') + test_target.OpenZipFileAtPath(source_path, mode='r').AndReturn(source) + test_target.OpenZipFileAtPath(dest_path, + compress=zipfile.ZIP_DEFLATED, + mode='w').AndReturn(dest) + self.my_mox.StubOutWithMock(os, 'rename') + os.rename(dest_path, source_path) + self.my_mox.StubOutWithMock(os, 'unlink') + os.unlink(source_path) + + self.my_mox.ReplayAll() + test_target.RemoveLastFile() + self.my_mox.VerifyAll() + + def CreateZipSource(self): + """Create a mock zip sourec object. + + Read should only be called once, because the second file is the one + being removed. + + Returns: + A configured mocked + """ + + source_zip = self.my_mox.CreateMock(zipfile.ZipFile) + source_zip.infolist().AndReturn([self.files['file1'], self.files['file1']]) + source_zip.infolist().AndReturn([self.files['file1'], self.files['file1']]) + source_zip.read(self.files['file1'].filename).AndReturn( + self.files['file1'].contents) + source_zip.close() + return source_zip + + def CreateZipDestination(self): + """Create mock destination zip. + + Write should only be called once, because there are two files in the + source zip and we expect the second to be removed. + + Returns: + A configured mocked + """ + + dest_zip = mox.MockObject(zipfile.ZipFile) + dest_zip.writestr(self.files['file1'].filename, + self.files['file1'].contents) + dest_zip.close() + return dest_zip + + +class FixArchiveTests(unittest.TestCase): + """Tests for the DirectoryZipper.FixArchive method.""" + + def setUp(self): + """Create a mock file object.""" + self.my_mox = mox.Mox() + self.file1 = BagOfParts() + self.file1.filename = 'file1.txt' + self.file1.contents = 'This is a test file' + + def tearDown(self): + """Unset any mocks that we've created.""" + self.my_mox.UnsetStubs() + + def _InitMultiFileData(self): + """Create an array of mock file objects. + + Create three mock file objects that we can use for testing. + """ + self.multi_file_dir = [] + + file1 = BagOfParts() + file1.filename = 'file1.txt' + file1.contents = 'kjaskl;jkdjfkja;kjsnbvjnvnbuewklriujalvjsd' + self.multi_file_dir.append(file1) + + file2 = BagOfParts() + file2.filename = 'file2.txt' + file2.contents = ('He entered the room and there in the center, it was.' + ' Looking upon the thing, suddenly he could not remember' + ' whether he had actually seen it before or whether' + ' his memory of it was merely the effect of something' + ' so often being imagined that it had long since become ' + ' manifest in his mind.') + self.multi_file_dir.append(file2) + + file3 = BagOfParts() + file3.filename = 'file3.txt' + file3.contents = 'Whoa, what is \'file2.txt\' all about?' + self.multi_file_dir.append(file3) + + def testSingleFileArchive(self): + """Test behavior of FixArchive when the archive has a single member. + + We expect that when this method is called with an archive that has a + single member that it will return False and unlink the archive. + """ + test_target = divide_and_compress.DirectoryZipper( + ''.join([os.getcwd(), '/']), 'dummy', 1024*1024, True) + self.my_mox.StubOutWithMock(test_target, 'OpenZipFileAtPath') + test_target.OpenZipFileAtPath( + ''.join([os.getcwd(), '/0.zip']), mode='r').AndReturn( + self.CreateSingleFileMock()) + self.my_mox.StubOutWithMock(os, 'unlink') + os.unlink(''.join([os.getcwd(), '/0.zip'])) + self.my_mox.ReplayAll() + self.assertEqual(False, test_target.FixArchive('SIZE')) + self.my_mox.VerifyAll() + + def CreateSingleFileMock(self): + """Create a mock ZipFile object for testSingleFileArchive. + + We just need it to return a single member infolist twice + + Returns: + A configured mock object + """ + mock_zip = self.my_mox.CreateMock(zipfile.ZipFile) + mock_zip.infolist().AndReturn([self.file1]) + mock_zip.infolist().AndReturn([self.file1]) + mock_zip.close() + return mock_zip + + def testMultiFileArchive(self): + """Test behavior of DirectoryZipper.FixArchive with a multi-file archive. + + We expect that FixArchive will rename the old archive, adding '-old' before + '.zip', read all the members except the last one of '-old' into a new + archive with the same name as the original, and then unlink the '-old' copy + """ + test_target = divide_and_compress.DirectoryZipper( + ''.join([os.getcwd(), '/']), 'dummy', 1024*1024, True) + self.my_mox.StubOutWithMock(test_target, 'OpenZipFileAtPath') + test_target.OpenZipFileAtPath( + ''.join([os.getcwd(), '/0.zip']), mode='r').AndReturn( + self.CreateMultiFileMock()) + self.my_mox.StubOutWithMock(test_target, 'RemoveLastFile') + test_target.RemoveLastFile(''.join([os.getcwd(), '/0.zip'])) + self.my_mox.StubOutWithMock(os, 'stat') + os.stat(''.join([os.getcwd(), '/0.zip'])).AndReturn([49302]) + self.my_mox.StubOutWithMock(stat, 'ST_SIZE') + stat.ST_SIZE = 0 + self.my_mox.ReplayAll() + self.assertEqual(True, test_target.FixArchive('SIZE')) + self.my_mox.VerifyAll() + + def CreateMultiFileMock(self): + """Create mock ZipFile object for use with testMultiFileArchive. + + The mock just needs to return the infolist mock that is prepared in + InitMultiFileData() + + Returns: + A configured mock object + """ + self._InitMultiFileData() + mock_zip = self.my_mox.CreateMock(zipfile.ZipFile) + mock_zip.infolist().AndReturn(self.multi_file_dir) + mock_zip.close() + return mock_zip + + +class AddFileToArchiveTest(unittest.TestCase): + """Test behavior of method to add a file to an archive.""" + + def setUp(self): + """Setup the arguments for the DirectoryZipper object.""" + self.my_mox = mox.Mox() + self.output_dir = '%s/' % os.getcwd() + self.file_to_add = 'file.txt' + self.input_dir = '/foo/bar/baz/' + + def tearDown(self): + self.my_mox.UnsetStubs() + + def testAddFileToArchive(self): + """Test the DirectoryZipper.AddFileToArchive method. + + We are testing a pretty trivial method, we just expect it to look at the + file its adding, so that it possible can through out a warning. + """ + test_target = divide_and_compress.DirectoryZipper(self.output_dir, + self.input_dir, + 1024*1024, True) + self.my_mox.StubOutWithMock(test_target, 'OpenZipFileAtPath') + archive_mock = self.CreateArchiveMock() + test_target.OpenZipFileAtPath( + ''.join([self.output_dir, '0.zip']), + compress=zipfile.ZIP_DEFLATED).AndReturn(archive_mock) + self.StubOutOsModule() + self.my_mox.ReplayAll() + test_target.AddFileToArchive(''.join([self.input_dir, self.file_to_add]), + zipfile.ZIP_DEFLATED) + self.my_mox.VerifyAll() + + def StubOutOsModule(self): + """Create a mock for the os.path and os.stat objects. + + Create a stub that will return the type (file or directory) and size of the + object that is to be added. + """ + self.my_mox.StubOutWithMock(os.path, 'isfile') + os.path.isfile(''.join([self.input_dir, self.file_to_add])).AndReturn(True) + self.my_mox.StubOutWithMock(os, 'stat') + os.stat(''.join([self.input_dir, self.file_to_add])).AndReturn([39480]) + self.my_mox.StubOutWithMock(stat, 'ST_SIZE') + stat.ST_SIZE = 0 + + def CreateArchiveMock(self): + """Create a mock ZipFile for use with testAddFileToArchive. + + Just verify that write is called with the file we expect and that the + archive is closed after the file addition + + Returns: + A configured mock object + """ + archive_mock = self.my_mox.CreateMock(zipfile.ZipFile) + archive_mock.write(''.join([self.input_dir, self.file_to_add]), + self.file_to_add) + archive_mock.close() + return archive_mock + + +class CompressDirectoryTest(unittest.TestCase): + """Test the master method of the class. + + Testing with the following directory structure. + /dir1/ + /dir1/file1.txt + /dir1/file2.txt + /dir1/dir2/ + /dir1/dir2/dir3/ + /dir1/dir2/dir4/ + /dir1/dir2/dir4/file3.txt + /dir1/dir5/ + /dir1/dir5/file4.txt + /dir1/dir5/file5.txt + /dir1/dir5/file6.txt + /dir1/dir5/file7.txt + /dir1/dir6/ + /dir1/dir6/file8.txt + + file1.txt., file2.txt, file3.txt should be in 0.zip + file4.txt should be in 1.zip + file5.txt, file6.txt should be in 2.zip + file7.txt will not be stored since it will be too large compressed + file8.txt should b in 3.zip + """ + + def setUp(self): + """Setup all the mocks for this test.""" + self.my_mox = mox.Mox() + + self.base_dir = '/dir1' + self.output_path = '/out_dir/' + self.test_target = divide_and_compress.DirectoryZipper( + self.output_path, self.base_dir, 1024*1024, True) + + self.InitArgLists() + self.InitOsDotPath() + self.InitArchiveIsValid() + self.InitWriteIndexRecord() + self.InitAddFileToArchive() + + def tearDown(self): + self.my_mox.UnsetStubs() + + def testCompressDirectory(self): + """Test the DirectoryZipper.CompressDirectory method.""" + self.my_mox.ReplayAll() + for arguments in self.argument_lists: + self.test_target.CompressDirectory(None, arguments[0], arguments[1]) + self.my_mox.VerifyAll() + + def InitAddFileToArchive(self): + """Setup mock for DirectoryZipper.AddFileToArchive. + + Make sure that the files are added in the order we expect. + """ + self.my_mox.StubOutWithMock(self.test_target, 'AddFileToArchive') + self.test_target.AddFileToArchive('/dir1/file1.txt', zipfile.ZIP_DEFLATED) + self.test_target.AddFileToArchive('/dir1/file2.txt', zipfile.ZIP_DEFLATED) + self.test_target.AddFileToArchive('/dir1/dir2/dir4/file3.txt', + zipfile.ZIP_DEFLATED) + self.test_target.AddFileToArchive('/dir1/dir5/file4.txt', + zipfile.ZIP_DEFLATED) + self.test_target.AddFileToArchive('/dir1/dir5/file4.txt', + zipfile.ZIP_DEFLATED) + self.test_target.AddFileToArchive('/dir1/dir5/file5.txt', + zipfile.ZIP_DEFLATED) + self.test_target.AddFileToArchive('/dir1/dir5/file5.txt', + zipfile.ZIP_DEFLATED) + self.test_target.AddFileToArchive('/dir1/dir5/file6.txt', + zipfile.ZIP_DEFLATED) + self.test_target.AddFileToArchive('/dir1/dir5/file7.txt', + zipfile.ZIP_DEFLATED) + self.test_target.AddFileToArchive('/dir1/dir5/file7.txt', + zipfile.ZIP_DEFLATED) + self.test_target.AddFileToArchive('/dir1/dir6/file8.txt', + zipfile.ZIP_DEFLATED) + + def InitWriteIndexRecord(self): + """Setup mock for DirectoryZipper.WriteIndexRecord.""" + self.my_mox.StubOutWithMock(self.test_target, 'WriteIndexRecord') + + # we are trying to compress 8 files, but we should only attempt to + # write an index record 7 times, because one file is too large to be stored + self.test_target.WriteIndexRecord().AndReturn(True) + self.test_target.WriteIndexRecord().AndReturn(False) + self.test_target.WriteIndexRecord().AndReturn(False) + self.test_target.WriteIndexRecord().AndReturn(True) + self.test_target.WriteIndexRecord().AndReturn(True) + self.test_target.WriteIndexRecord().AndReturn(False) + self.test_target.WriteIndexRecord().AndReturn(True) + + def InitArchiveIsValid(self): + """Mock out DirectoryZipper.ArchiveIsValid and DirectoryZipper.FixArchive. + + Mock these methods out such that file1, file2, and file3 go into one + archive. file4 then goes into the next archive, file5 and file6 in the + next, file 7 should appear too large to compress into an archive, and + file8 goes into the final archive + """ + self.my_mox.StubOutWithMock(self.test_target, 'ArchiveIsValid') + self.my_mox.StubOutWithMock(self.test_target, 'FixArchive') + self.test_target.ArchiveIsValid().AndReturn(True) + self.test_target.ArchiveIsValid().AndReturn(True) + self.test_target.ArchiveIsValid().AndReturn(True) + + # should be file4.txt + self.test_target.ArchiveIsValid().AndReturn(False) + self.test_target.FixArchive('SIZE').AndReturn(True) + self.test_target.ArchiveIsValid().AndReturn(True) + + # should be file5.txt + self.test_target.ArchiveIsValid().AndReturn(False) + self.test_target.FixArchive('SIZE').AndReturn(True) + self.test_target.ArchiveIsValid().AndReturn(True) + self.test_target.ArchiveIsValid().AndReturn(True) + + # should be file7.txt + self.test_target.ArchiveIsValid().AndReturn(False) + self.test_target.FixArchive('SIZE').AndReturn(True) + self.test_target.ArchiveIsValid().AndReturn(False) + self.test_target.FixArchive('SIZE').AndReturn(False) + self.test_target.ArchiveIsValid().AndReturn(True) + + def InitOsDotPath(self): + """Mock out os.path.isfile. + + Mock this out so the things we want to appear as files appear as files and + the things we want to appear as directories appear as directories. Also + make sure that the order of file visits is as we expect (which is why + InAnyOrder isn't used here). + """ + self.my_mox.StubOutWithMock(os.path, 'isfile') + os.path.isfile('/dir1/dir2').AndReturn(False) + os.path.isfile('/dir1/dir5').AndReturn(False) + os.path.isfile('/dir1/dir6').AndReturn(False) + os.path.isfile('/dir1/file1.txt').AndReturn(True) + os.path.isfile('/dir1/file2.txt').AndReturn(True) + os.path.isfile('/dir1/dir2/dir3').AndReturn(False) + os.path.isfile('/dir1/dir2/dir4').AndReturn(False) + os.path.isfile('/dir1/dir2/dir4/file3.txt').AndReturn(True) + os.path.isfile('/dir1/dir5/file4.txt').AndReturn(True) + os.path.isfile('/dir1/dir5/file4.txt').AndReturn(True) + os.path.isfile('/dir1/dir5/file5.txt').AndReturn(True) + os.path.isfile('/dir1/dir5/file5.txt').AndReturn(True) + os.path.isfile('/dir1/dir5/file6.txt').AndReturn(True) + os.path.isfile('/dir1/dir5/file7.txt').AndReturn(True) + os.path.isfile('/dir1/dir5/file7.txt').AndReturn(True) + os.path.isfile('/dir1/dir6/file8.txt').AndReturn(True) + + def InitArgLists(self): + """Create the directory path => directory contents mappings.""" + self.argument_lists = [] + self.argument_lists.append(['/dir1', + ['file1.txt', 'file2.txt', 'dir2', 'dir5', + 'dir6']]) + self.argument_lists.append(['/dir1/dir2', ['dir3', 'dir4']]) + self.argument_lists.append(['/dir1/dir2/dir3', []]) + self.argument_lists.append(['/dir1/dir2/dir4', ['file3.txt']]) + self.argument_lists.append(['/dir1/dir5', + ['file4.txt', 'file5.txt', 'file6.txt', + 'file7.txt']]) + self.argument_lists.append(['/dir1/dir6', ['file8.txt']]) + +if __name__ == '__main__': + unittest.main()