diff --git a/.vscode/launch.json b/.vscode/launch.json index 486f77f16b23..3ee6613ad13a 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -29,7 +29,7 @@ "runtimeArgs": [ "--harmony" ], - "program": "./out/client/debugger/main.js", + "program": "./out/client/debugger/vs/vsdebugger.js", "stopOnEntry": false, "args": [ "--server=4711" diff --git a/README.md b/README.md index 5ff9cad71139..a175951fb0fc 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ # Python -Linting, Debugging, Intellisense, auto-completion, code formatting, snippets, unit testing, and more. +Linting, Debugging (multi threaded, web apps), Intellisense, auto-completion, code formatting, snippets, unit testing, and more. Works on both Windows and Mac. ##Features @@ -8,13 +8,16 @@ Works on both Windows and Mac. * Code formatting (autopep8, yapf, with config files) * Renaming, Viewing references, Going to definitions, Go to Symbols * View signature and similar by hovering over a function or method -* Debugging with support for local & global variables, arguments, expressions, watch window, stack information, break points +* Debugging with support for local variables, arguments, expressions, watch window, stack information, break points +* Debugging Multiple threads (Web Applications, etc) and expanding values in watch windows is supported on Windows * Unit testing (unittests and nosetests, with config files) * Sorting imports * Snippets -## Issues and Feature Requests +## Issues and Feature Requests [Github Issues](https://github.com/DonJayamanne/pythonVSCode/issues) +* Remote Debugging (coming soon) +* Improved debugging for Mac (in development) ## Feature Details (with confiuration) * IDE Features @@ -28,6 +31,13 @@ Works on both Windows and Mac. * - Support for docstring * - Ability to include custom module paths (e.g. include paths for libraries like Google App Engine, etc) * - - Use the setting python.autoComplete.extraPaths = [] +* - - For instance getting autocomplete/intellisense for Google App Engine, add the following to your settings file: +```json +"python.autoComplete.extraPaths": [ + "C:/Program Files (x86)/Google/google_appengine", + "C:/Program Files (x86)/Google/google_appengine/lib" + ] +``` * Code formatting * - Use either yapf or autopep8 for code formatting (defaults to autopep8) * - auutopep8 configuration files supported @@ -44,7 +54,9 @@ Works on both Windows and Mac. * - Evaluate Expressions * - Step through code (Step in, Step out, Continue) * - Add/remove break points -* - Local variables, Global variables and arguments (experimental, still needs some polishing) +* - Local variables and arguments +* - Multiple Threads and Web Applications (such as Flask) (only Windows at this stage) +* - Expanding values (viewing children, properties, etc) again only Windows at this Stage * Unit Testing * - unittests (default is on) * - nosetests (default is off) @@ -59,6 +71,10 @@ Works on both Windows and Mac. ![Image of Renaming and Find all References](https://raw.githubusercontent.com/DonJayamanne/pythonVSCode/master/images/rename.gif) +![Image of Debugging](https://raw.githubusercontent.com/DonJayamanne/pythonVSCode/master/images/standardDebugging.gif) + +![Image of Multi Threaded Debugging](https://raw.githubusercontent.com/DonJayamanne/pythonVSCode/master/images/flaskDebugging.gif) + ## Requirements * Python is installed on the current system * - Path to python can be configured @@ -78,6 +94,12 @@ Works on both Windows and Mac. ## Change Log +### Version 0.2.0 +* Improved debugger for Windows, with support for Multi threading, debugging Multi-threaded apps, Web Applications, expanding properties, etc +* Added support for relative paths for extra paths in additional libraries for Auto Complete +* Fixed a bug where paths to custom Python versions weren't respected by the previous (PDB) debugger +* NOTE: PDB Debugger is still supported + ### Version 0.1.3 * Fixed linting when using pylint diff --git a/images/flaskDebugging.gif b/images/flaskDebugging.gif new file mode 100644 index 000000000000..bfe120a9da80 Binary files /dev/null and b/images/flaskDebugging.gif differ diff --git a/images/standardDebugging.gif b/images/standardDebugging.gif new file mode 100644 index 000000000000..4053d6e811d4 Binary files /dev/null and b/images/standardDebugging.gif differ diff --git a/package.json b/package.json index 964cb5935240..617fc89c01b2 100644 --- a/package.json +++ b/package.json @@ -1,8 +1,8 @@ { "name": "python", "displayName": "Python", - "description": "Linting, Debugging, Intellisense, auto-completion, code formatting, snippets, and more.", - "version": "0.1.3", + "description": "Linting, Debugging (multi-threaded), Intellisense, auto-completion, code formatting, snippets, and more.", + "version": "0.2.0", "publisher": "donjayamanne", "license": "SEE LICENSE IN LICENSE or README.MD", "homepage": "https://github.com/DonJayamanne/pythonVSCode/blob/master/README.md", @@ -54,13 +54,13 @@ "debuggers": [ { "type": "python", - "label": "Python", + "label": "Python (pdb)", "enableBreakpointsFor": { "languageIds": [ "python" ] }, - "program": "./out/client/debugger/main.js", + "program": "./out/client/debugger/pdb/debuggerMain.js", "runtime": "node", "configurationAttributes": { "launch": { @@ -73,23 +73,76 @@ "description": "Workspace relative path to a text file.", "default": "__init__.py" }, + "pythonPath": { + "type": "string", + "description": "Path (fully qualified) to python executable. Use this if you want to use a custom pthon executable version.", + "default": "" + }, "stopOnEntry": { "type": "boolean", "description": "Automatically stop after launch.", "default": true + }, + "args": { + "type": "array", + "description": "List of arguments for the program", + "default": [] } } } }, "initialConfigurations": [ { - "name": "Python", + "name": "Python (Pdb)", "type": "python", "request": "launch", "program": "__init__.py", "stopOnEntry": true } ] + }, + { + "type": "python_windows", + "label": "Python (Windows)", + "enableBreakpointsFor": { + "languageIds": [ + "python" + ] + }, + "program": "./out/client/debugger/vs/VSDebugger.js", + "runtime": "node", + "configurationAttributes": { + "launch": { + "required": [ + "program" + ], + "properties": { + "program": { + "type": "string", + "description": "Workspace relative path to a text file.", + "default": "__init__.py" + }, + "pythonPath": { + "type": "string", + "description": "Path (fully qualified) to python executable. Use this if you want to use a custom pthon executable version.", + "default": "" + }, + "args": { + "type": "array", + "description": "List of arguments for the program", + "default": [] + } + } + } + }, + "initialConfigurations": [ + { + "name": "Python (Windows)", + "type": "python_windows", + "request": "launch", + "program": "__init__.py" + } + ] } ], "configuration": { @@ -224,7 +277,7 @@ "type": "string", "default": "yapf", "description": "Path to yapf, you can use a custom version of yapf by modifying this setting to include the full path." - }, + }, "python.autoComplete.extraPaths": { "type": "array", "default": [], @@ -253,6 +306,7 @@ "compile": "node ./node_modules/vscode/bin/compile -watch -p ./ && installServerIntoExtension ./out ./src/server/package.json ./src/server/tsconfig.json" }, "dependencies": { + "long": "^3.0.3", "named-js-regexp": "^1.3.1", "tmp": "0.0.28", "vscode-debugadapter": "^1.0.1", diff --git a/pythonFiles/PythonTools/visualstudio_ipython_repl.py b/pythonFiles/PythonTools/visualstudio_ipython_repl.py new file mode 100644 index 000000000000..d09e3988d9db --- /dev/null +++ b/pythonFiles/PythonTools/visualstudio_ipython_repl.py @@ -0,0 +1,423 @@ +# Python Tools for Visual Studio +# Copyright(c) Microsoft Corporation +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the License); you may not use +# this file except in compliance with the License. You may obtain a copy of the +# License at http://www.apache.org/licenses/LICENSE-2.0 +# +# THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +# OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +# IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +# MERCHANTABLITY OR NON-INFRINGEMENT. +# +# See the Apache Version 2.0 License for specific language governing +# permissions and limitations under the License. + +"""Implements REPL support over IPython/ZMQ for VisualStudio""" + +__author__ = "Microsoft Corporation " +__version__ = "3.0.0.0" + +import re +import sys +from visualstudio_py_repl import BasicReplBackend, ReplBackend, UnsupportedReplException, _command_line_to_args_list +from visualstudio_py_util import to_bytes +try: + import thread +except: + import _thread as thread # Renamed as Py3k + +from base64 import decodestring + +try: + import IPython +except ImportError: + exc_value = sys.exc_info()[1] + raise UnsupportedReplException('IPython mode requires IPython 0.11 or later: ' + str(exc_value)) + +def is_ipython_versionorgreater(major, minor): + """checks if we are at least a specific IPython version""" + match = re.match('(\d+).(\d+)', IPython.__version__) + if match: + groups = match.groups() + if int(groups[0]) > major: + return True + elif int(groups[0]) == major: + return int(groups[1]) >= minor + + return False + +remove_escapes = re.compile(r'\x1b[^m]*m') + +try: + if is_ipython_versionorgreater(3, 0): + from IPython.kernel import KernelManager + from IPython.kernel.channels import HBChannel + from IPython.kernel.threaded import (ThreadedZMQSocketChannel, ThreadedKernelClient as KernelClient) + ShellChannel = StdInChannel = IOPubChannel = ThreadedZMQSocketChannel + elif is_ipython_versionorgreater(1, 0): + from IPython.kernel import KernelManager, KernelClient + from IPython.kernel.channels import ShellChannel, HBChannel, StdInChannel, IOPubChannel + else: + import IPython.zmq + KernelClient = object # was split out from KernelManager in 1.0 + from IPython.zmq.kernelmanager import (KernelManager, + ShellSocketChannel as ShellChannel, + SubSocketChannel as IOPubChannel, + StdInSocketChannel as StdInChannel, + HBSocketChannel as HBChannel) + + from IPython.utils.traitlets import Type +except ImportError: + exc_value = sys.exc_info()[1] + raise UnsupportedReplException(str(exc_value)) + + +# TODO: SystemExit exceptions come back to us as strings, can we automatically exit when ones raised somehow? + +##### +# Channels which forward events + +# Description of the messaging protocol +# http://ipython.scipy.org/doc/manual/html/development/messaging.html + + +class DefaultHandler(object): + def unknown_command(self, content): + import pprint + print('unknown command ' + str(type(self))) + pprint.pprint(content) + + def call_handlers(self, msg): + # msg_type: + # execute_reply + msg_type = 'handle_' + msg['msg_type'] + + getattr(self, msg_type, self.unknown_command)(msg['content']) + +class VsShellChannel(DefaultHandler, ShellChannel): + + def handle_execute_reply(self, content): + # we could have a payload here... + payload = content['payload'] + + for item in payload: + data = item.get('data') + if data is not None: + try: + # Could be named km.sub_channel for very old IPython, but + # those versions should not put 'data' in this payload + write_data = self._vs_backend.km.iopub_channel.write_data + except AttributeError: + pass + else: + write_data(data) + continue + + output = item.get('text', None) + if output is not None: + self._vs_backend.write_stdout(output) + self._vs_backend.send_command_executed() + + def handle_inspect_reply(self, content): + self.handle_object_info_reply(content) + + def handle_object_info_reply(self, content): + self._vs_backend.object_info_reply = content + self._vs_backend.members_lock.release() + + def handle_complete_reply(self, content): + self._vs_backend.complete_reply = content + self._vs_backend.members_lock.release() + + def handle_kernel_info_reply(self, content): + self._vs_backend.write_stdout(content['banner']) + + +class VsIOPubChannel(DefaultHandler, IOPubChannel): + def call_handlers(self, msg): + # only output events from our session or no sessions + # https://pytools.codeplex.com/workitem/1622 + parent = msg.get('parent_header') + if not parent or parent.get('session') == self.session.session: + msg_type = 'handle_' + msg['msg_type'] + getattr(self, msg_type, self.unknown_command)(msg['content']) + + def handle_display_data(self, content): + # called when user calls display() + data = content.get('data', None) + + if data is not None: + self.write_data(data) + + def handle_stream(self, content): + stream_name = content['name'] + if is_ipython_versionorgreater(3, 0): + output = content['text'] + else: + output = content['data'] + if stream_name == 'stdout': + self._vs_backend.write_stdout(output) + elif stream_name == 'stderr': + self._vs_backend.write_stderr(output) + # TODO: stdin can show up here, do we echo that? + + def handle_execute_result(self, content): + self.handle_execute_output(content) + + def handle_execute_output(self, content): + # called when an expression statement is printed, we treat + # identical to stream output but it always goes to stdout + output = content['data'] + execution_count = content['execution_count'] + self._vs_backend.execution_count = execution_count + 1 + self._vs_backend.send_prompt('\r\nIn [%d]: ' % (execution_count + 1), ' ' + ('.' * (len(str(execution_count + 1)) + 2)) + ': ', False) + self.write_data(output, execution_count) + + def write_data(self, data, execution_count = None): + output_xaml = data.get('application/xaml+xml', None) + if output_xaml is not None: + try: + if isinstance(output_xaml, str) and sys.version_info[0] >= 3: + output_xaml = output_xaml.encode('ascii') + self._vs_backend.write_xaml(decodestring(output_xaml)) + self._vs_backend.write_stdout('\n') + return + except: + pass + + output_png = data.get('image/png', None) + if output_png is not None: + try: + if isinstance(output_png, str) and sys.version_info[0] >= 3: + output_png = output_png.encode('ascii') + self._vs_backend.write_png(decodestring(output_png)) + self._vs_backend.write_stdout('\n') + return + except: + pass + + output_str = data.get('text/plain', None) + if output_str is not None: + if execution_count is not None: + if '\n' in output_str: + output_str = '\n' + output_str + output_str = 'Out[' + str(execution_count) + ']: ' + output_str + + self._vs_backend.write_stdout(output_str) + self._vs_backend.write_stdout('\n') + return + + def handle_error(self, content): + # TODO: this includes escape sequences w/ color, we need to unescape that + ename = content['ename'] + evalue = content['evalue'] + tb = content['traceback'] + self._vs_backend.write_stderr('\n'.join(tb)) + self._vs_backend.write_stdout('\n') + + def handle_execute_input(self, content): + # just a rebroadcast of the command to be executed, can be ignored + self._vs_backend.execution_count += 1 + self._vs_backend.send_prompt('\r\nIn [%d]: ' % (self._vs_backend.execution_count), ' ' + ('.' * (len(str(self._vs_backend.execution_count)) + 2)) + ': ', False) + pass + + def handle_status(self, content): + pass + + # Backwards compat w/ 0.13 + handle_pyin = handle_execute_input + handle_pyout = handle_execute_output + handle_pyerr = handle_error + + +class VsStdInChannel(DefaultHandler, StdInChannel): + def handle_input_request(self, content): + # queue this to another thread so we don't block the channel + def read_and_respond(): + value = self._vs_backend.read_line() + + self.input(value) + + thread.start_new_thread(read_and_respond, ()) + + +class VsHBChannel(DefaultHandler, HBChannel): + pass + + +class VsKernelManager(KernelManager, KernelClient): + shell_channel_class = Type(VsShellChannel) + if is_ipython_versionorgreater(1, 0): + iopub_channel_class = Type(VsIOPubChannel) + else: + sub_channel_class = Type(VsIOPubChannel) + stdin_channel_class = Type(VsStdInChannel) + hb_channel_class = Type(VsHBChannel) + + +class IPythonBackend(ReplBackend): + def __init__(self, mod_name = '__main__', launch_file = None): + ReplBackend.__init__(self) + self.launch_file = launch_file + self.mod_name = mod_name + self.km = VsKernelManager() + + if is_ipython_versionorgreater(0, 13): + # http://pytools.codeplex.com/workitem/759 + # IPython stopped accepting the ipython flag and switched to launcher, the new + # default is what we want though. + self.km.start_kernel(**{'extra_arguments': self.get_extra_arguments()}) + else: + self.km.start_kernel(**{'ipython': True, 'extra_arguments': self.get_extra_arguments()}) + self.km.start_channels() + self.exit_lock = thread.allocate_lock() + self.exit_lock.acquire() # used as an event + self.members_lock = thread.allocate_lock() + self.members_lock.acquire() + + self.km.shell_channel._vs_backend = self + self.km.stdin_channel._vs_backend = self + if is_ipython_versionorgreater(1, 0): + self.km.iopub_channel._vs_backend = self + else: + self.km.sub_channel._vs_backend = self + self.km.hb_channel._vs_backend = self + self.execution_count = 1 + + def get_extra_arguments(self): + if sys.version <= '2.': + return [unicode('--pylab=inline')] + return ['--pylab=inline'] + + def execute_file_as_main(self, filename, arg_string): + f = open(filename, 'rb') + try: + contents = f.read().replace(to_bytes("\r\n"), to_bytes("\n")) + finally: + f.close() + args = [filename] + _command_line_to_args_list(arg_string) + code = ''' +import sys +sys.argv = %(args)r +__file__ = %(filename)r +del sys +exec(compile(%(contents)r, %(filename)r, 'exec')) +''' % {'filename' : filename, 'contents':contents, 'args': args} + + self.run_command(code, True) + + def execution_loop(self): + # launch the startup script if one has been specified + if self.launch_file: + self.execute_file_as_main(self.launch_file, None) + + # we've got a bunch of threads setup for communication, we just block + # here until we're requested to exit. + self.send_prompt('\r\nIn [1]: ', ' ...: ', False) + self.exit_lock.acquire() + + def run_command(self, command, silent = False): + if is_ipython_versionorgreater(3, 0): + self.km.execute(command, silent) + else: + self.km.shell_channel.execute(command, silent) + + def execute_file(self, filename, args): + self.execute_file_as_main(filename, args) + + def exit_process(self): + self.exit_lock.release() + + def get_members(self, expression): + """returns a tuple of the type name, instance members, and type members""" + text = expression + '.' + if is_ipython_versionorgreater(3, 0): + self.km.complete(text) + else: + self.km.shell_channel.complete(text, text, 1) + + self.members_lock.acquire() + + reply = self.complete_reply + + res = {} + text_len = len(text) + for member in reply['matches']: + res[member[text_len:]] = 'object' + + return ('unknown', res, {}) + + def get_signatures(self, expression): + """returns doc, args, vargs, varkw, defaults.""" + + if is_ipython_versionorgreater(3, 0): + self.km.inspect(expression, None, 2) + else: + self.km.shell_channel.object_info(expression) + + self.members_lock.acquire() + + reply = self.object_info_reply + if is_ipython_versionorgreater(3, 0): + data = reply['data'] + text = data['text/plain'] + text = remove_escapes.sub('', text) + return [(text, (), None, None, [])] + else: + argspec = reply['argspec'] + defaults = argspec['defaults'] + if defaults is not None: + defaults = [repr(default) for default in defaults] + else: + defaults = [] + return [(reply['docstring'], argspec['args'], argspec['varargs'], argspec['varkw'], defaults)] + + def interrupt_main(self): + """aborts the current running command""" + self.km.interrupt_kernel() + + def set_current_module(self, module): + pass + + def get_module_names(self): + """returns a list of module names""" + return [] + + def flush(self): + pass + + def init_debugger(self): + from os import path + self.run_command(''' +def __visualstudio_debugger_init(): + import sys + sys.path.append(''' + repr(path.dirname(__file__)) + ''') + import visualstudio_py_debugger + new_thread = visualstudio_py_debugger.new_thread() + sys.settrace(new_thread.trace_func) + visualstudio_py_debugger.intercept_threads(True) + +__visualstudio_debugger_init() +del __visualstudio_debugger_init +''', True) + + def attach_process(self, port, debugger_id): + self.run_command(''' +def __visualstudio_debugger_attach(): + import visualstudio_py_debugger + + def do_detach(): + visualstudio_py_debugger.DETACH_CALLBACKS.remove(do_detach) + + visualstudio_py_debugger.DETACH_CALLBACKS.append(do_detach) + visualstudio_py_debugger.attach_process(''' + str(port) + ''', ''' + repr(debugger_id) + ''', report = True, block = True) + +__visualstudio_debugger_attach() +del __visualstudio_debugger_attach +''', True) + +class IPythonBackendWithoutPyLab(IPythonBackend): + def get_extra_arguments(self): + return [] diff --git a/pythonFiles/PythonTools/visualstudio_py_debugger.py b/pythonFiles/PythonTools/visualstudio_py_debugger.py new file mode 100644 index 000000000000..3f53e91155d8 --- /dev/null +++ b/pythonFiles/PythonTools/visualstudio_py_debugger.py @@ -0,0 +1,2501 @@ +# Python Tools for Visual Studio +# Copyright(c) Microsoft Corporation +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the License); you may not use +# this file except in compliance with the License. You may obtain a copy of the +# License at http://www.apache.org/licenses/LICENSE-2.0 +# +# THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +# OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +# IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +# MERCHANTABLITY OR NON-INFRINGEMENT. +# +# See the Apache Version 2.0 License for specific language governing +# permissions and limitations under the License. + +from __future__ import with_statement + +__author__ = "Microsoft Corporation " +__version__ = "3.0.0.0" + +# This module MUST NOT import threading in global scope. This is because in a direct (non-ptvsd) +# attach scenario, it is loaded on the injected debugger attach thread, and if threading module +# hasn't been loaded already, it will assume that the thread on which it is being loaded is the +# main thread. This will cause issues when the thread goes away after attach completes. +_threading = None + +import sys +import ctypes +try: + import thread +except ImportError: + import _thread as thread +import socket +import struct +import weakref +import traceback +import types +import bisect +from os import path +import ntpath +import runpy +import datetime +from codecs import BOM_UTF8 + +try: + # In the local attach scenario, visualstudio_py_util is injected into globals() + # by PyDebugAttach before loading this module, and cannot be imported. + _vspu = visualstudio_py_util +except: + try: + import visualstudio_py_util as _vspu + except ImportError: + import ptvsd.visualstudio_py_util as _vspu + +to_bytes = _vspu.to_bytes +exec_file = _vspu.exec_file +exec_module = _vspu.exec_module +exec_code = _vspu.exec_code +read_bytes = _vspu.read_bytes +read_int = _vspu.read_int +read_string = _vspu.read_string +write_bytes = _vspu.write_bytes +write_int = _vspu.write_int +write_string = _vspu.write_string +safe_repr = _vspu.SafeRepr() + +try: + # In the local attach scenario, visualstudio_py_repl is injected into globals() + # by PyDebugAttach before loading this module, and cannot be imported. + _vspr = visualstudio_py_repl +except: + try: + import visualstudio_py_repl as _vspr + except ImportError: + import ptvsd.visualstudio_py_repl as _vspr + +try: + import stackless +except ImportError: + stackless = None + +try: + xrange +except: + xrange = range + +if sys.platform == 'cli': + import clr + from System.Runtime.CompilerServices import ConditionalWeakTable + IPY_SEEN_MODULES = ConditionalWeakTable[object, object]() + +# Import encodings early to avoid import on the debugger thread, which may cause deadlock +from encodings import utf_8 + +# WARNING: Avoid imports beyond this point, specifically on the debugger thread, as this may cause +# deadlock where the debugger thread performs an import while a user thread has the import lock + +# save start_new_thread so we can call it later, we'll intercept others calls to it. + +debugger_dll_handle = None +DETACHED = True +def thread_creator(func, args, kwargs = {}, *extra_args): + if not isinstance(args, tuple): + # args is not a tuple. This may be because we have become bound to a + # class, which has offset our arguments by one. + if isinstance(kwargs, tuple): + func, args = args, kwargs + kwargs = extra_args[0] if len(extra_args) > 0 else {} + + return _start_new_thread(new_thread_wrapper, (func, args, kwargs)) + +_start_new_thread = thread.start_new_thread +THREADS = {} +THREADS_LOCK = thread.allocate_lock() +MODULES = [] + +BREAK_ON_SYSTEMEXIT_ZERO = False +DEBUG_STDLIB = False +DJANGO_DEBUG = False + +# Py3k compat - alias unicode to str +try: + unicode +except: + unicode = str + +# A value of a synthesized child. The string is passed through to the variable list, and type is not displayed at all. +class SynthesizedValue(object): + def __init__(self, repr_value='', len_value=None): + self.repr_value = repr_value + self.len_value = len_value + def __repr__(self): + return self.repr_value + def __len__(self): + return self.len_value + +# Specifies list of files not to debug. Can be extended by other modules +# (the REPL does this for $attach support and not stepping into the REPL). +DONT_DEBUG = [path.normcase(__file__), path.normcase(_vspu.__file__)] +if sys.version_info >= (3, 3): + DONT_DEBUG.append(path.normcase('')) +if sys.version_info >= (3, 5): + DONT_DEBUG.append(path.normcase('')) + +# Contains information about all breakpoints in the process. Keys are line numbers on which +# there are breakpoints in any file, and values are dicts. For every line number, the +# corresponding dict contains all the breakpoints that fall on that line. The keys in that +# dict are tuples of the form (filename, breakpoint_id), each entry representing a single +# breakpoint, and values are BreakpointInfo objects. +# +# For example, given the following breakpoints: +# +# 1. In 'main.py' at line 10. +# 2. In 'main.py' at line 20. +# 3. In 'module.py' at line 10. +# +# the contents of BREAKPOINTS would be: +# {10: {('main.py', 1): ..., ('module.py', 3): ...}, 20: {('main.py', 2): ... }} +BREAKPOINTS = {} + +# Contains information about all pending (i.e. not yet bound) breakpoints in the process. +# Elements are BreakpointInfo objects. +PENDING_BREAKPOINTS = set() + +# Must be in sync with enum PythonBreakpointConditionKind in PythonBreakpoint.cs +BREAKPOINT_CONDITION_ALWAYS = 0 +BREAKPOINT_CONDITION_WHEN_TRUE = 1 +BREAKPOINT_CONDITION_WHEN_CHANGED = 2 + +# Must be in sync with enum PythonBreakpointPassCountKind in PythonBreakpoint.cs +BREAKPOINT_PASS_COUNT_ALWAYS = 0 +BREAKPOINT_PASS_COUNT_EVERY = 1 +BREAKPOINT_PASS_COUNT_WHEN_EQUAL = 2 +BREAKPOINT_PASS_COUNT_WHEN_EQUAL_OR_GREATER = 3 + +class BreakpointInfo(object): + __slots__ = [ + 'breakpoint_id', 'filename', 'lineno', 'condition_kind', 'condition', + 'pass_count_kind', 'pass_count', 'is_bound', 'last_condition_value', + 'hit_count' + ] + + # For "when changed" breakpoints, this is used as the initial value of last_condition_value, + # such that it is guaranteed to not compare equal to any other value that it will get later. + _DUMMY_LAST_VALUE = object() + + def __init__(self, breakpoint_id, filename, lineno, condition_kind, condition, pass_count_kind, pass_count): + self.breakpoint_id = breakpoint_id + self.filename = filename + self.lineno = lineno + self.condition_kind = condition_kind + self.condition = condition + self.pass_count_kind = pass_count_kind + self.pass_count = pass_count + self.is_bound = False + self.last_condition_value = BreakpointInfo._DUMMY_LAST_VALUE + self.hit_count = 0 + + @staticmethod + def find_by_id(breakpoint_id): + for line, bp_dict in BREAKPOINTS.items(): + for (filename, bp_id), bp in bp_dict.items(): + if bp_id == breakpoint_id: + return bp + return None + +# lock for calling .send on the socket +send_lock = thread.allocate_lock() + +class _SendLockContextManager(object): + """context manager for send lock. Handles both acquiring/releasing the + send lock as well as detaching the debugger if the remote process + is disconnected""" + + def __enter__(self): + # mark that we're about to do socket I/O so we won't deliver + # debug events when we're debugging the standard library + cur_thread = get_thread_from_id(thread.get_ident()) + if cur_thread is not None: + cur_thread.is_sending = True + + send_lock.acquire() + + def __exit__(self, exc_type, exc_value, tb): + send_lock.release() + + # start sending debug events again + cur_thread = get_thread_from_id(thread.get_ident()) + if cur_thread is not None: + cur_thread.is_sending = False + + if exc_type is not None: + detach_threads() + detach_process() + # swallow the exception, we're no longer debugging + return True + +_SendLockCtx = _SendLockContextManager() + +SEND_BREAK_COMPLETE = False + +STEPPING_OUT = -1 # first value, we decrement below this +STEPPING_NONE = 0 +STEPPING_BREAK = 1 +STEPPING_LAUNCH_BREAK = 2 +STEPPING_ATTACH_BREAK = 3 +STEPPING_INTO = 4 +STEPPING_OVER = 5 # last value, we increment past this. + +USER_STEPPING = (STEPPING_OUT, STEPPING_INTO, STEPPING_OVER) + +FRAME_KIND_NONE = 0 +FRAME_KIND_PYTHON = 1 +FRAME_KIND_DJANGO = 2 + +DJANGO_BUILTINS = {'True': True, 'False': False, 'None': None} + +PYTHON_EVALUATION_RESULT_REPR_KIND_NORMAL = 0 # regular repr and hex repr (if applicable) for the evaluation result; length is len(result) +PYTHON_EVALUATION_RESULT_REPR_KIND_RAW = 1 # repr is raw representation of the value - see TYPES_WITH_RAW_REPR; length is len(repr) +PYTHON_EVALUATION_RESULT_REPR_KIND_RAWLEN = 2 # same as above, but only the length is reported, not the actual value + +PYTHON_EVALUATION_RESULT_EXPANDABLE = 1 +PYTHON_EVALUATION_RESULT_METHOD_CALL = 2 +PYTHON_EVALUATION_RESULT_SIDE_EFFECTS = 4 +PYTHON_EVALUATION_RESULT_RAW = 8 +PYTHON_EVALUATION_RESULT_HAS_RAW_REPR = 16 + +# Don't show attributes of these types if they come from the class (assume they are methods). +METHOD_TYPES = ( + types.FunctionType, + types.MethodType, + types.BuiltinFunctionType, + type("".__repr__), # method-wrapper +) + +# repr() for these types can be used as input for eval() to get the original value. +# float is intentionally not included because it is not always round-trippable (e.g inf, nan). +TYPES_WITH_ROUND_TRIPPING_REPR = set((type(None), int, bool, str, unicode)) +if sys.version[0] == '3': + TYPES_WITH_ROUND_TRIPPING_REPR.add(bytes) +else: + TYPES_WITH_ROUND_TRIPPING_REPR.add(long) + +# repr() for these types can be used as input for eval() to get the original value, provided that the same is true for all their elements. +COLLECTION_TYPES_WITH_ROUND_TRIPPING_REPR = set((tuple, list, set, frozenset)) + +# eval(repr(x)), but optimized for common types for which it is known that result == x. +def eval_repr(x): + def is_repr_round_tripping(x): + # Do exact type checks here - subclasses can override __repr__. + if type(x) in TYPES_WITH_ROUND_TRIPPING_REPR: + return True + elif type(x) in COLLECTION_TYPES_WITH_ROUND_TRIPPING_REPR: + # All standard sequence types are round-trippable if their elements are. + return all((is_repr_round_tripping(item) for item in x)) + else: + return False + if is_repr_round_tripping(x): + return x + else: + return eval(repr(x), {}) + +# key is type, value is function producing the raw repr +TYPES_WITH_RAW_REPR = { + unicode: (lambda s: s) +} + +# bytearray is 2.6+ +try: + # getfilesystemencoding is used here because it effectively corresponds to the notion of "locale encoding": + # current ANSI codepage on Windows, LC_CTYPE on Linux, UTF-8 on OS X - which is exactly what we want. + TYPES_WITH_RAW_REPR[bytearray] = lambda b: b.decode(sys.getfilesystemencoding(), 'ignore') +except: + pass + +if sys.version[0] == '3': + TYPES_WITH_RAW_REPR[bytes] = TYPES_WITH_RAW_REPR[bytearray] +else: + TYPES_WITH_RAW_REPR[str] = TYPES_WITH_RAW_REPR[unicode] + +if sys.version[0] == '3': + # work around a crashing bug on CPython 3.x where they take a hard stack overflow + # we'll never see this exception but it'll allow us to keep our try/except handler + # the same across all versions of Python + class StackOverflowException(Exception): pass +else: + StackOverflowException = RuntimeError + +ASBR = to_bytes('ASBR') +SETL = to_bytes('SETL') +THRF = to_bytes('THRF') +DETC = to_bytes('DETC') +NEWT = to_bytes('NEWT') +EXTT = to_bytes('EXTT') +EXIT = to_bytes('EXIT') +EXCP = to_bytes('EXCP') +MODL = to_bytes('MODL') +STPD = to_bytes('STPD') +BRKS = to_bytes('BRKS') +BRKF = to_bytes('BRKF') +BRKH = to_bytes('BRKH') +BRKC = to_bytes('BRKC') +BKHC = to_bytes('BKHC') +LOAD = to_bytes('LOAD') +EXCE = to_bytes('EXCE') +EXCR = to_bytes('EXCR') +CHLD = to_bytes('CHLD') +OUTP = to_bytes('OUTP') +REQH = to_bytes('REQH') +LAST = to_bytes('LAST') + +def get_thread_from_id(id): + THREADS_LOCK.acquire() + try: + return THREADS.get(id) + finally: + THREADS_LOCK.release() + +def should_send_frame(frame): + return (frame is not None and + frame.f_code not in DEBUG_ENTRYPOINTS and + path.normcase(frame.f_code.co_filename) not in DONT_DEBUG) + +KNOWN_DIRECTORIES = set((None, '')) +KNOWN_ZIPS = set() + +def is_file_in_zip(filename): + parent, name = path.split(path.abspath(filename)) + if parent in KNOWN_DIRECTORIES: + return False + elif parent in KNOWN_ZIPS: + return True + elif path.isdir(parent): + KNOWN_DIRECTORIES.add(parent) + return False + else: + KNOWN_ZIPS.add(parent) + return True + +def lookup_builtin(name, frame): + try: + return frame.f_builtins.get(bits) + except: + # http://ironpython.codeplex.com/workitem/30908 + builtins = frame.f_globals['__builtins__'] + if not isinstance(builtins, dict): + builtins = builtins.__dict__ + return builtins.get(name) + +def lookup_local(frame, name): + bits = name.split('.') + obj = frame.f_locals.get(bits[0]) or frame.f_globals.get(bits[0]) or lookup_builtin(bits[0], frame) + bits.pop(0) + while bits and obj is not None and type(obj) is types.ModuleType: + obj = getattr(obj, bits.pop(0), None) + return obj + +if sys.version_info[0] >= 3: + _EXCEPTIONS_MODULE = 'builtins' +else: + _EXCEPTIONS_MODULE = 'exceptions' + +def get_exception_name(exc_type): + if exc_type.__module__ == _EXCEPTIONS_MODULE: + return exc_type.__name__ + else: + return exc_type.__module__ + '.' + exc_type.__name__ + +# These constants come from Visual Studio - enum_EXCEPTION_STATE +BREAK_MODE_NEVER = 0 +BREAK_MODE_ALWAYS = 1 +BREAK_MODE_UNHANDLED = 32 + +BREAK_TYPE_NONE = 0 +BREAK_TYPE_UNHANDLED = 1 +BREAK_TYPE_HANDLED = 2 + +class ExceptionBreakInfo(object): + BUILT_IN_HANDLERS = { + path.normcase(''): ((None, None, '*'),), + path.normcase('build\\bdist.win32\\egg\\pkg_resources.py'): ((None, None, '*'),), + path.normcase('build\\bdist.win-amd64\\egg\\pkg_resources.py'): ((None, None, '*'),), + } + + def __init__(self): + self.default_mode = BREAK_MODE_UNHANDLED + self.break_on = { } + self.handler_cache = dict(self.BUILT_IN_HANDLERS) + self.handler_lock = thread.allocate_lock() + self.add_exception('exceptions.IndexError', BREAK_MODE_NEVER) + self.add_exception('builtins.IndexError', BREAK_MODE_NEVER) + self.add_exception('exceptions.KeyError', BREAK_MODE_NEVER) + self.add_exception('builtins.KeyError', BREAK_MODE_NEVER) + self.add_exception('exceptions.AttributeError', BREAK_MODE_NEVER) + self.add_exception('builtins.AttributeError', BREAK_MODE_NEVER) + self.add_exception('exceptions.StopIteration', BREAK_MODE_NEVER) + self.add_exception('builtins.StopIteration', BREAK_MODE_NEVER) + self.add_exception('exceptions.GeneratorExit', BREAK_MODE_NEVER) + self.add_exception('builtins.GeneratorExit', BREAK_MODE_NEVER) + + def clear(self): + self.default_mode = BREAK_MODE_UNHANDLED + self.break_on.clear() + self.handler_cache = dict(self.BUILT_IN_HANDLERS) + + def should_break(self, thread, ex_type, ex_value, trace): + probe_stack() + name = get_exception_name(ex_type) + mode = self.break_on.get(name, self.default_mode) + break_type = BREAK_TYPE_NONE + if mode & BREAK_MODE_ALWAYS: + if self.is_handled(thread, ex_type, ex_value, trace): + break_type = BREAK_TYPE_HANDLED + else: + break_type = BREAK_TYPE_UNHANDLED + elif (mode & BREAK_MODE_UNHANDLED) and not self.is_handled(thread, ex_type, ex_value, trace): + break_type = BREAK_TYPE_UNHANDLED + + if break_type: + if issubclass(ex_type, SystemExit): + if not BREAK_ON_SYSTEMEXIT_ZERO: + if not ex_value or (isinstance(ex_value, SystemExit) and not ex_value.code): + break_type = BREAK_TYPE_NONE + + return break_type + + def is_handled(self, thread, ex_type, ex_value, trace): + if trace is None: + # get out if we didn't get a traceback + return False + + if trace.tb_next is not None: + if should_send_frame(trace.tb_next.tb_frame) and should_debug_code(trace.tb_next.tb_frame.f_code): + # don't break if this is not the top of the traceback, + # unless the previous frame was not debuggable + return True + + cur_frame = trace.tb_frame + + while should_send_frame(cur_frame) and cur_frame.f_code is not None and cur_frame.f_code.co_filename is not None: + filename = path.normcase(cur_frame.f_code.co_filename) + if is_file_in_zip(filename): + # File is in a zip, so assume it handles exceptions + return True + + if not is_same_py_file(filename, __file__): + handlers = self.handler_cache.get(filename) + + if handlers is None: + # req handlers for this file from the debug engine + self.handler_lock.acquire() + + with _SendLockCtx: + write_bytes(conn, REQH) + write_string(conn, filename) + + # wait for the handler data to be received + self.handler_lock.acquire() + self.handler_lock.release() + + handlers = self.handler_cache.get(filename) + + if handlers is None: + # no code available, so assume unhandled + return False + + line = cur_frame.f_lineno + for line_start, line_end, expressions in handlers: + if line_start is None or line_start <= line < line_end: + if '*' in expressions: + return True + + for text in expressions: + try: + res = lookup_local(cur_frame, text) + if res is not None and issubclass(ex_type, res): + return True + except: + pass + + cur_frame = cur_frame.f_back + + return False + + def add_exception(self, name, mode=BREAK_MODE_UNHANDLED): + if name.startswith(_EXCEPTIONS_MODULE + '.'): + name = name[len(_EXCEPTIONS_MODULE) + 1:] + self.break_on[name] = mode + +BREAK_ON = ExceptionBreakInfo() + +def probe_stack(depth = 10): + """helper to make sure we have enough stack space to proceed w/o corrupting + debugger state.""" + if depth == 0: + return + probe_stack(depth - 1) + +PREFIXES = [path.normcase(sys.prefix)] +# If we're running in a virtual env, DEBUG_STDLIB should respect this too. +if hasattr(sys, 'base_prefix'): + PREFIXES.append(path.normcase(sys.base_prefix)) +if hasattr(sys, 'real_prefix'): + PREFIXES.append(path.normcase(sys.real_prefix)) + +def should_debug_code(code): + if not code or not code.co_filename: + return False + + filename = path.normcase(code.co_filename) + if not DEBUG_STDLIB: + for prefix in PREFIXES: + if prefix != '' and filename.startswith(prefix): + return False + + for dont_debug_file in DONT_DEBUG: + if is_same_py_file(filename, dont_debug_file): + return False + + if is_file_in_zip(filename): + # file in inside an egg or zip, so we can't debug it + return False + + return True + +attach_lock = thread.allocate() +attach_sent_break = False + +local_path_to_vs_path = {} + +def breakpoint_path_match(vs_path, local_path): + vs_path_norm = path.normcase(vs_path) + local_path_norm = path.normcase(local_path) + if local_path_to_vs_path.get(local_path_norm) == vs_path_norm: + return True + + # Walk the local filesystem from local_path up, matching agains win_path component by component, + # and stop when we no longer see an __init__.py. This should give a reasonably close approximation + # of matching the package name. + while True: + local_path, local_name = path.split(local_path) + vs_path, vs_name = ntpath.split(vs_path) + # Match the last component in the path. If one or both components are unavailable, then + # we have reached the root on the corresponding path without successfully matching. + if not local_name or not vs_name or path.normcase(local_name) != path.normcase(vs_name): + return False + # If we have an __init__.py, this module was inside the package, and we still need to match + # thatpackage, so walk up one level and keep matching. Otherwise, we've walked as far as we + # needed to, and matched all names on our way, so this is a match. + if not path.exists(path.join(local_path, '__init__.py')): + break + + local_path_to_vs_path[local_path_norm] = vs_path_norm + return True + +def update_all_thread_stacks(blocking_thread = None, check_is_blocked = True): + THREADS_LOCK.acquire() + all_threads = list(THREADS.values()) + THREADS_LOCK.release() + + for cur_thread in all_threads: + if cur_thread is blocking_thread: + continue + + cur_thread._block_starting_lock.acquire() + if not check_is_blocked or not cur_thread._is_blocked: + # release the lock, we're going to run user code to evaluate the frames + cur_thread._block_starting_lock.release() + + frames = cur_thread.get_frame_list() + + # re-acquire the lock and make sure we're still not blocked. If so send + # the frame list. + cur_thread._block_starting_lock.acquire() + if not check_is_blocked or not cur_thread._is_blocked: + cur_thread.send_frame_list(frames) + + cur_thread._block_starting_lock.release() + +DJANGO_BREAKPOINTS = {} + +class DjangoBreakpointInfo(object): + def __init__(self, filename): + self._line_locations = None + self.filename = filename + self.breakpoints = {} + + def add_breakpoint(self, lineno, brkpt_id): + self.breakpoints[lineno] = brkpt_id + + def remove_breakpoint(self, lineno): + del self.breakpoints[lineno] + + @property + def line_locations(self): + if self._line_locations is None: + # we need to calculate our line number offset information + try: + contents = open(self.filename, 'rb') + except: + # file not available, locked, etc... + pass + else: + with contents: + line_info = [] + file_len = 0 + for line in contents: + if not line_info and line.startswith(BOM_UTF8): + line = line[3:] # Strip the BOM, Django seems to ignore this... + file_len += len(line) + line_info.append(file_len) + contents.close() + self._line_locations = line_info + + return self._line_locations + + def get_line_range(self, start, end): + line_locs = self.line_locations + if line_locs is not None: + low_line = bisect.bisect_right(line_locs, start) + hi_line = bisect.bisect_right(line_locs, end) + + return low_line, hi_line + + return (None, None) + + def should_break(self, start, end): + low_line, hi_line = self.get_line_range(start, end) + if low_line is not None and hi_line is not None: + # low_line/hi_line is 0 based, self.breakpoints is 1 based + for i in xrange(low_line+1, hi_line+2): + bkpt_id = self.breakpoints.get(i) + if bkpt_id is not None: + return True, bkpt_id + + return False, 0 + +def get_django_frame_source(frame): + if frame.f_code.co_name == 'render': + self_obj = frame.f_locals.get('self', None) + if self_obj is not None and type(self_obj).__name__ != 'TextNode': + source_obj = getattr(self_obj, 'source', None) + if source_obj is not None: + return source_obj + + return None + +class ModuleExitFrame(object): + def __init__(self, real_frame): + self.real_frame = real_frame + self.f_lineno = real_frame.f_lineno + 1 + + def __getattr__(self, name): + return getattr(self.real_frame, name) + +class Thread(object): + def __init__(self, id = None): + if id is not None: + self.id = id + else: + self.id = thread.get_ident() + self._events = {'call' : self.handle_call, + 'line' : self.handle_line, + 'return' : self.handle_return, + 'exception' : self.handle_exception, + 'c_call' : self.handle_c_call, + 'c_return' : self.handle_c_return, + 'c_exception' : self.handle_c_exception, + } + self.cur_frame = None + self.stepping = STEPPING_NONE + self.unblock_work = None + self._block_lock = thread.allocate_lock() + self._block_lock.acquire() + self._block_starting_lock = thread.allocate_lock() + self._is_blocked = False + self._is_working = False + self.stopped_on_line = None + self.detach = False + self.trace_func = self.trace_func # replace self.trace_func w/ a bound method so we don't need to re-create these regularly + self.prev_trace_func = None + self.trace_func_stack = [] + self.reported_process_loaded = False + self.django_stepping = None + self.is_sending = False + + # stackless changes + if stackless is not None: + self._stackless_attach() + + if sys.platform == 'cli': + self.frames = [] + + if sys.platform == 'cli': + # workaround an IronPython bug where we're sometimes missing the back frames + # http://ironpython.codeplex.com/workitem/31437 + def push_frame(self, frame): + self.cur_frame = frame + self.frames.append(frame) + + def pop_frame(self): + self.frames.pop() + self.cur_frame = self.frames[-1] + else: + def push_frame(self, frame): + self.cur_frame = frame + + def pop_frame(self): + self.cur_frame = self.cur_frame.f_back + + def _stackless_attach(self): + try: + stackless.tasklet.trace_function + except AttributeError: + # the tasklets need to be traced on a case by case basis + # sys.trace needs to be called within their calling context + def __call__(tsk, *args, **kwargs): + f = tsk.tempval + def new_f(old_f, args, kwargs): + sys.settrace(self.trace_func) + try: + if old_f is not None: + return old_f(*args, **kwargs) + finally: + sys.settrace(None) + + tsk.tempval = new_f + stackless.tasklet.setup(tsk, f, args, kwargs) + return tsk + + def settrace(tsk, tb): + if hasattr(tsk.frame, "f_trace"): + tsk.frame.f_trace = tb + sys.settrace(tb) + + self.__oldstacklesscall__ = stackless.tasklet.__call__ + stackless.tasklet.settrace = settrace + stackless.tasklet.__call__ = __call__ + if sys.platform == 'cli': + self.frames = [] + + if sys.platform == 'cli': + # workaround an IronPython bug where we're sometimes missing the back frames + # http://ironpython.codeplex.com/workitem/31437 + def push_frame(self, frame): + self.cur_frame = frame + self.frames.append(frame) + + def pop_frame(self): + self.frames.pop() + self.cur_frame = self.frames[-1] + else: + def push_frame(self, frame): + self.cur_frame = frame + + def pop_frame(self): + self.cur_frame = self.cur_frame.f_back + + def context_dispatcher(self, old, new): + self.stepping = STEPPING_NONE + # for those tasklets that started before we started tracing + # we need to make sure that the trace is set by patching + # it in the context switch + if old and new: + if hasattr(new.frame, "f_trace") and not new.frame.f_trace: + sys.call_tracing(new.settrace,(self.trace_func,)) + + def _stackless_schedule_cb(self, prev, next): + current = stackless.getcurrent() + if not current: + return + current_tf = current.trace_function + + try: + current.trace_function = None + self.stepping = STEPPING_NONE + + # If the current frame has no trace function, we may need to get it + # from the previous frame, depending on how we ended up in the + # callback. + if current_tf is None: + f_back = current.frame.f_back + if f_back is not None: + current_tf = f_back.f_trace + + if next is not None: + # Assign our trace function to the current stack + f = next.frame + if next is current: + f = f.f_back + while f: + if isinstance(f, types.FrameType): + f.f_trace = self.trace_func + f = f.f_back + next.trace_function = self.trace_func + finally: + current.trace_function = current_tf + + def trace_func(self, frame, event, arg): + # If we're so far into process shutdown that sys is already gone, just stop tracing. + if sys is None: + return None + elif self.is_sending: + # https://pytools.codeplex.com/workitem/1864 + # we're currently doing I/O w/ the socket, we don't want to deliver + # any breakpoints or async breaks because we'll deadlock. Continue + # to return the trace function so all of our frames remain + # balanced. A better way to deal with this might be to do + # sys.settrace(None) when we take the send lock, but that's much + # more difficult because our send context manager is used both + # inside and outside of the trace function, and so is used when + # tracing is enabled and disabled, and so it's very easy to get our + # current frame tracking to be thrown off... + return self.trace_func + + try: + # if should_debug_code(frame.f_code) is not true during attach + # the current frame is None and a pop_frame will cause an exception and + # break the debugger + if self.cur_frame is None: + # happens during attach, we need frame for blocking + self.push_frame(frame) + if self.stepping == STEPPING_BREAK and should_debug_code(frame.f_code): + if self.detach: + if stackless is not None: + stackless.set_schedule_callback(None) + stackless.tasklet.__call__ = self.__oldstacklesscall__ + sys.settrace(None) + return None + + self.async_break() + + return self._events[event](frame, arg) + except (StackOverflowException, KeyboardInterrupt): + # stack overflow, disable tracing + return self.trace_func + + def handle_call(self, frame, arg): + self.push_frame(frame) + + if DJANGO_BREAKPOINTS: + source_obj = get_django_frame_source(frame) + if source_obj is not None: + origin, (start, end) = source_obj + + active_bps = DJANGO_BREAKPOINTS.get(origin.name.lower()) + should_break = False + if active_bps is not None: + should_break, bkpt_id = active_bps.should_break(start, end) + if should_break: + probe_stack() + update_all_thread_stacks(self) + self.block(lambda: (report_breakpoint_hit(bkpt_id, self.id), mark_all_threads_for_break(skip_thread = self))) + if not should_break and self.django_stepping: + self.django_stepping = None + self.stepping = STEPPING_OVER + self.block_maybe_attach() + + if frame.f_code.co_name == '' and frame.f_code.co_filename not in ['', '']: + probe_stack() + code, module = new_module(frame) + if not DETACHED: + report_module_load(module) + + # see if this module causes new break points to be bound + bound = set() + for pending_bp in PENDING_BREAKPOINTS: + if try_bind_break_point(code.co_filename, module, pending_bp): + bound.add(pending_bp) + PENDING_BREAKPOINTS.difference_update(bound) + + stepping = self.stepping + if stepping is not STEPPING_NONE and should_debug_code(frame.f_code): + if stepping == STEPPING_INTO: + # block when we hit the 1st line, not when we're on the function def + self.stepping = STEPPING_OVER + # empty stopped_on_line so that we will break even if it is + # the same line + self.stopped_on_line = None + elif stepping >= STEPPING_OVER: + self.stepping += 1 + elif stepping <= STEPPING_OUT: + self.stepping -= 1 + + if (sys.platform == 'cli' and + frame.f_code.co_name == '' and + not IPY_SEEN_MODULES.TryGetValue(frame.f_code)[0]): + IPY_SEEN_MODULES.Add(frame.f_code, None) + # work around IronPython bug - http://ironpython.codeplex.com/workitem/30127 + self.handle_line(frame, arg) + + # forward call to previous trace function, if any, saving old trace func for when we return + old_trace_func = self.prev_trace_func + if old_trace_func is not None: + self.trace_func_stack.append(old_trace_func) + self.prev_trace_func = None # clear first incase old_trace_func stack overflows + self.prev_trace_func = old_trace_func(frame, 'call', arg) + + return self.trace_func + + def should_block_on_frame(self, frame): + if not should_debug_code(frame.f_code): + return False + # It is still possible that we're somewhere in standard library code, but that code was invoked by our + # internal debugger machinery (e.g. socket.sendall or text encoding while tee'ing print output to VS). + # We don't want to block on any of that, either, so walk the stack and see if we hit debugger frames + # at some point below the non-debugger ones. + while frame is not None: + # There is usually going to be a debugger frame at the very bottom of the stack - the one that + # invoked user code on this thread when starting debugging. If we reached that, then everything + # above is user code, so we know that we do want to block here. + if frame.f_code in DEBUG_ENTRYPOINTS: + break + # Otherwise, check if it's some other debugger code. + filename = path.normcase(frame.f_code.co_filename) + is_debugger_frame = False + for debugger_file in DONT_DEBUG: + if is_same_py_file(filename, debugger_file): + # If it is, then the frames above it on the stack that we have just walked through + # were for debugger internal purposes, and we do not want to block here. + return False + frame = frame.f_back + return True + + def handle_line(self, frame, arg): + if not DETACHED: + # resolve whether step_complete and/or handling_breakpoints + step_complete = False + handle_breakpoints = True + stepping = self.stepping + if stepping is not STEPPING_NONE: # check for the common case of no stepping first... + if ((stepping == STEPPING_OVER or stepping == STEPPING_INTO) and frame.f_lineno != self.stopped_on_line): + if self.should_block_on_frame(frame): # don't step complete in our own debugger / non-user code + step_complete = True + elif stepping == STEPPING_LAUNCH_BREAK or stepping == STEPPING_ATTACH_BREAK: + # If launching rather than attaching, don't break into initial Python code needed to set things up + if stepping == STEPPING_LAUNCH_BREAK and (not MODULES or not self.should_block_on_frame(frame)): + handle_breakpoints = False + else: + step_complete = True + + # handle breakpoints + hit_bp_id = None + if BREAKPOINTS and handle_breakpoints: + bp = BREAKPOINTS.get(frame.f_lineno) + if bp is not None: + for (filename, bp_id), bp in bp.items(): + if filename != frame.f_code.co_filename: + # When the breakpoint is bound, the filename is updated to match co_filename of + # the module to which it was bound, so only exact matches are considered hits. + if bp.is_bound: + continue + # Otherwise, use relaxed path check that tries to handle differences between + # local and remote filesystems for remote scenarios: + if not breakpoint_path_match(filename, frame.f_code.co_filename): + continue + + # If we got here, filename and line number both match. + + # Check condition to see if we actually hit this breakpoint. + if bp.condition_kind != BREAKPOINT_CONDITION_ALWAYS: + try: + res = eval(bp.condition, frame.f_globals, frame.f_locals) + if bp.condition_kind == BREAKPOINT_CONDITION_WHEN_CHANGED: + last_val = bp.last_condition_value + bp.last_condition_value = res + if last_val == res: + # Condition didn't change, breakpoint not hit. + continue + else: + if not res: + # Condition isn't true, breakpoint not hit. + continue + except: + # If anything goes wrong while evaluating condition, breakpoint is hit. + pass + + # If we got here, then condition matched, and we need to update the hit count + # (even if we don't end up signaling the breakpoint because of pass count). + bp.hit_count += 1 + + # Check the new hit count against pass count. + if bp.pass_count_kind != BREAKPOINT_PASS_COUNT_ALWAYS: + pass_count_kind = bp.pass_count_kind + pass_count = bp.pass_count + hit_count = bp.hit_count + if pass_count_kind == BREAKPOINT_PASS_COUNT_EVERY: + if (hit_count % pass_count) != 0: + continue + elif pass_count_kind == BREAKPOINT_PASS_COUNT_WHEN_EQUAL: + if hit_count != pass_count: + continue + elif pass_count_kind == BREAKPOINT_PASS_COUNT_WHEN_EQUAL_OR_GREATER: + if hit_count < pass_count: + continue + + # If we got here, then condition and pass count both match, so we should notify VS. + hit_bp_id = bp_id + + # There may be other breakpoints for the same file/line, and we need to update + # their hit counts, too, so keep looping. If more than one is hit, it's fine, + # we will just signal the last one. + + if hit_bp_id is not None: + # handle case where both hitting a breakpoint and step complete by reporting the breakpoint + # if the reported breakpoint is a tracepoint, report the step complete if/when the tracepoint is auto-resumed + probe_stack() + update_all_thread_stacks(self) + self.block(lambda: (report_breakpoint_hit(hit_bp_id, self.id), mark_all_threads_for_break(skip_thread = self)), step_complete) + + elif step_complete: + self.block_maybe_attach() + + # forward call to previous trace function, if any, updating trace function appropriately + old_trace_func = self.prev_trace_func + if old_trace_func is not None: + self.prev_trace_func = None # clear first incase old_trace_func stack overflows + self.prev_trace_func = old_trace_func(frame, 'line', arg) + + return self.trace_func + + def handle_return(self, frame, arg): + self.pop_frame() + + if not DETACHED: + stepping = self.stepping + # only update stepping state when this frame is debuggable (matching handle_call) + if stepping is not STEPPING_NONE and should_debug_code(frame.f_code): + if stepping > STEPPING_OVER: + self.stepping -= 1 + elif stepping < STEPPING_OUT: + self.stepping += 1 + elif stepping in USER_STEPPING: + if self.cur_frame is None or frame.f_code.co_name == "" : + # only return to user code modules + if self.should_block_on_frame(frame): + # restore back the module frame for the step out of a module + self.push_frame(ModuleExitFrame(frame)) + self.stepping = STEPPING_NONE + update_all_thread_stacks(self) + self.block(lambda: (report_step_finished(self.id), mark_all_threads_for_break(skip_thread = self))) + self.pop_frame() + elif self.should_block_on_frame(self.cur_frame): + # if we're returning into non-user code then don't block in the + # non-user code, wait until we hit user code again + self.stepping = STEPPING_NONE + update_all_thread_stacks(self) + self.block(lambda: (report_step_finished(self.id), mark_all_threads_for_break(skip_thread = self))) + + # forward call to previous trace function, if any + old_trace_func = self.prev_trace_func + if old_trace_func is not None: + old_trace_func(frame, 'return', arg) + + # restore previous frames trace function if there is one + if self.trace_func_stack: + self.prev_trace_func = self.trace_func_stack.pop() + + def handle_exception(self, frame, arg): + if self.stepping == STEPPING_ATTACH_BREAK: + self.block_maybe_attach() + + if not DETACHED and should_debug_code(frame.f_code): + break_type = BREAK_ON.should_break(self, *arg) + if break_type: + update_all_thread_stacks(self) + self.block(lambda: report_exception(frame, arg, self.id, break_type)) + + # forward call to previous trace function, if any, updating the current trace function + # with a new one if available + old_trace_func = self.prev_trace_func + if old_trace_func is not None: + self.prev_trace_func = old_trace_func(frame, 'exception', arg) + + return self.trace_func + + def handle_c_call(self, frame, arg): + # break points? + pass + + def handle_c_return(self, frame, arg): + # step out of ? + pass + + def handle_c_exception(self, frame, arg): + pass + + def block_maybe_attach(self): + will_block_now = True + if self.stepping == STEPPING_ATTACH_BREAK: + # only one thread should send the attach break in + attach_lock.acquire() + global attach_sent_break + if attach_sent_break: + will_block_now = False + attach_sent_break = True + attach_lock.release() + + probe_stack() + stepping = self.stepping + self.stepping = STEPPING_NONE + def block_cond(): + if will_block_now: + if stepping == STEPPING_OVER or stepping == STEPPING_INTO: + report_step_finished(self.id) + return mark_all_threads_for_break(skip_thread = self) + else: + if not DETACHED: + if stepping == STEPPING_ATTACH_BREAK: + self.reported_process_loaded = True + return report_process_loaded(self.id) + update_all_thread_stacks(self) + self.block(block_cond) + + def async_break(self): + def async_break_send(): + with _SendLockCtx: + sent_break_complete = False + global SEND_BREAK_COMPLETE + if SEND_BREAK_COMPLETE == True or SEND_BREAK_COMPLETE == self.id: + # multiple threads could be sending this... + SEND_BREAK_COMPLETE = False + sent_break_complete = True + write_bytes(conn, ASBR) + write_int(conn, self.id) + + if sent_break_complete: + # if we have threads which have not broken yet capture their frame list and + # send it now. If they block we'll send an updated (and possibly more accurate - if + # there are any thread locals) list of frames. + update_all_thread_stacks(self) + + self.stepping = STEPPING_NONE + self.block(async_break_send) + + def block(self, block_lambda, keep_stopped_on_line = False): + """blocks the current thread until the debugger resumes it""" + assert not self._is_blocked + #assert self.id == thread.get_ident(), 'wrong thread identity' + str(self.id) + ' ' + str(thread.get_ident()) # we should only ever block ourselves + + # send thread frames before we block + self.enum_thread_frames_locally() + + if not keep_stopped_on_line: + self.stopped_on_line = self.cur_frame.f_lineno + + # need to synchronize w/ sending the reason we're blocking + self._block_starting_lock.acquire() + self._is_blocked = True + block_lambda() + self._block_starting_lock.release() + + while not DETACHED: + self._block_lock.acquire() + if self.unblock_work is None: + break + + # the debugger wants us to do something, do it, and then block again + self._is_working = True + self.unblock_work() + self.unblock_work = None + self._is_working = False + + self._block_starting_lock.acquire() + assert self._is_blocked + self._is_blocked = False + self._block_starting_lock.release() + + def unblock(self): + """unblocks the current thread allowing it to continue to run""" + assert self._is_blocked + assert self.id != thread.get_ident() # only someone else should unblock us + + self._block_lock.release() + + def schedule_work(self, work): + self.unblock_work = work + self.unblock() + + def run_on_thread(self, text, cur_frame, execution_id, frame_kind, repr_kind = PYTHON_EVALUATION_RESULT_REPR_KIND_NORMAL): + self._block_starting_lock.acquire() + + if not self._is_blocked: + report_execution_error('', execution_id) + elif not self._is_working: + self.schedule_work(lambda : self.run_locally(text, cur_frame, execution_id, frame_kind, repr_kind)) + else: + report_execution_error('', execution_id) + + self._block_starting_lock.release() + + def run_on_thread_no_report(self, text, cur_frame, frame_kind): + self._block_starting_lock.acquire() + + if not self._is_blocked: + pass + elif not self._is_working: + self.schedule_work(lambda : self.run_locally_no_report(text, cur_frame, frame_kind)) + else: + pass + + self._block_starting_lock.release() + + def enum_child_on_thread(self, text, cur_frame, execution_id, frame_kind): + self._block_starting_lock.acquire() + if not self._is_working and self._is_blocked: + self.schedule_work(lambda : self.enum_child_locally(text, cur_frame, execution_id, frame_kind)) + self._block_starting_lock.release() + else: + self._block_starting_lock.release() + report_children(execution_id, []) + + def get_locals(self, cur_frame, frame_kind): + if frame_kind == FRAME_KIND_DJANGO: + locs = {} + # iterate going forward, so later items replace earlier items + for d in cur_frame.f_locals['context'].dicts: + # hasattr check to defend against someone passing a bad dictionary value + # and us breaking the app. + if hasattr(d, 'keys') and d != DJANGO_BUILTINS: + for key in d.keys(): + locs[key] = d[key] + else: + locs = cur_frame.f_locals + return locs + + def locals_to_fast(self, frame): + try: + ltf = ctypes.pythonapi.PyFrame_LocalsToFast + ltf.argtypes = [ctypes.py_object, ctypes.c_int] + ltf(frame, 1) + except: + pass + + def compile(self, text, cur_frame): + try: + code = compile(text, '', 'eval') + except: + code = compile(text, '', 'exec') + return code + + def run_locally(self, text, cur_frame, execution_id, frame_kind, repr_kind = PYTHON_EVALUATION_RESULT_REPR_KIND_NORMAL): + try: + code = self.compile(text, cur_frame) + res = eval(code, cur_frame.f_globals, self.get_locals(cur_frame, frame_kind)) + self.locals_to_fast(cur_frame) + # Report any updated variable values first + self.enum_thread_frames_locally() + report_execution_result(execution_id, res, repr_kind) + except: + # Report any updated variable values first + self.enum_thread_frames_locally() + report_execution_exception(execution_id, sys.exc_info()) + + def run_locally_no_report(self, text, cur_frame, frame_kind): + code = self.compile(text, cur_frame) + res = eval(code, cur_frame.f_globals, self.get_locals(cur_frame, frame_kind)) + self.locals_to_fast(cur_frame) + sys.displayhook(res) + + def enum_child_locally(self, expr, cur_frame, execution_id, frame_kind): + try: + code = compile(expr, cur_frame.f_code.co_name, 'eval') + res = eval(code, cur_frame.f_globals, self.get_locals(cur_frame, frame_kind)) + + children = [] # [(name, expression, value, flags)] + + # Process attributes. + + cls_dir = set(dir(type(res))) + res_dict = getattr(res, '__dict__', {}) + res_slots = set(getattr(res, '__slots__', ())) + + for attr_name in dir(res): + try: + # Skip special attributes. + if attr_name.startswith('__') and attr_name.endswith('__'): + continue + attr_value = getattr(res, attr_name) + # If it comes from the class and is not shadowed by any instance attribute, filter it out if it looks like a method. + if attr_name in cls_dir and attr_name not in res_dict and attr_name not in res_slots: + if isinstance(attr_value, METHOD_TYPES): + continue + children.append((attr_name, expr + '.' + attr_name, attr_value, 0)) + except: + # Skip this attribute if we can't process it. + pass + + # Process items, if this is a collection. + + try: + if hasattr(res, '__iter__') and iter(res) is res: + # An iterable object that is its own iterator - iterators, generators, enumerate() etc. These can only be iterated once, so + # don't try to iterate them immediately. Instead, provide a child item that will do so when expanded, to give user full control. + children.append(('Results View', 'tuple(' + expr + ')', SynthesizedValue('Expanding the Results View will run the iterator'), PYTHON_EVALUATION_RESULT_METHOD_CALL | PYTHON_EVALUATION_RESULT_SIDE_EFFECTS)) + enum = () + elif isinstance(res, dict) or (hasattr(res, 'items') and hasattr(res, 'has_key')): + # Dictionary-like object. + try: + enum = res.viewitems() + enum_expr = expr + '.viewitems()' + children.append(('viewitems()', enum_expr, SynthesizedValue(), PYTHON_EVALUATION_RESULT_METHOD_CALL)) + except: + enum = res.items() + enum_expr = expr + '.items()' + children.append(('items()', enum_expr, SynthesizedValue(), PYTHON_EVALUATION_RESULT_METHOD_CALL)) + enum_var = '(k, v)' + enum = enumerate(enum) + else: + # Indexable or enumerable object. + enum = enumerate(enumerate(res)) + enum_expr = expr + enum_var = 'v' + except: + enum = () + + for index, (key, item) in enum: + try: + if len(children) > 10000: + # Report at most 10000 items. + children.append(('[...]', None, 'Evaluation halted because sequence has too many items', 0)) + break + + key_repr = safe_repr(key) + + # Some objects are enumerable but not indexable, or repr(key) is not a valid Python expression. For those, we + # cannot use obj[key] to get the item by its key, and have to retrieve it by index from enumerate() instead. + try: + item_by_key = res[eval_repr(key)] + use_index = item is not item_by_key + except: + use_index = True + else: + use_index = False + + item_name = '[' + key_repr + ']' + if use_index: + item_expr = 'next((v for i, %s in enumerate(%s) if i == %s))' % (enum_var, enum_expr, index) + else: + item_expr = expr + item_name + + children.append((item_name, item_expr, item, 0)) + + except: + # Skip this item if we can't process it. + pass + + report_children(execution_id, children) + + except: + report_children(execution_id, []) + + def get_frame_list(self): + frames = [] + cur_frame = self.cur_frame + + while should_send_frame(cur_frame): + # calculate the ending line number + lineno = cur_frame.f_code.co_firstlineno + try: + linetable = cur_frame.f_code.co_lnotab + except: + try: + lineno = cur_frame.f_code.Span.End.Line + except: + lineno = -1 + else: + for line_incr in linetable[1::2]: + if sys.version >= '3': + lineno += line_incr + else: + lineno += ord(line_incr) + + frame_locals = cur_frame.f_locals + var_names = cur_frame.f_code.co_varnames + + source_obj = None + if DJANGO_DEBUG: + source_obj = get_django_frame_source(cur_frame) + if source_obj is not None: + frame_locals = self.get_locals(cur_frame, FRAME_KIND_DJANGO) + var_names = frame_locals + + if source_obj is not None: + process_globals_in_functions = False + elif frame_locals is cur_frame.f_globals: + var_names = frame_locals + process_globals_in_functions = False + else: + process_globals_in_functions = True + + # collect frame locals + vars = [] + treated = set() + self.collect_variables(vars, frame_locals, var_names, treated) + if process_globals_in_functions: + # collect closed over variables used locally (frame_locals not already treated based on var_names) + self.collect_variables(vars, frame_locals, frame_locals, treated) + # collect globals used locally, skipping undefined found in builtins + f_globals = cur_frame.f_globals + if f_globals: # ensure globals to work with (IPy may have None for cur_frame.f_globals for frames within stdlib) + self.collect_variables(vars, f_globals, cur_frame.f_code.co_names, treated, skip_unknown = True) + + frame_info = None + + if source_obj is not None: + origin, (start, end) = source_obj + + filename = str(origin) + bp_info = DJANGO_BREAKPOINTS.get(filename.lower()) + if bp_info is None: + DJANGO_BREAKPOINTS[filename.lower()] = bp_info = DjangoBreakpointInfo(filename) + + low_line, hi_line = bp_info.get_line_range(start, end) + if low_line is not None and hi_line is not None: + frame_kind = FRAME_KIND_DJANGO + frame_info = ( + low_line + 1, + hi_line + 1, + low_line + 1, + cur_frame.f_code.co_name, + str(origin), + 0, + vars, + FRAME_KIND_DJANGO, + get_code_filename(cur_frame.f_code), + cur_frame.f_lineno + ) + + if frame_info is None: + frame_info = ( + cur_frame.f_code.co_firstlineno, + lineno, + cur_frame.f_lineno, + cur_frame.f_code.co_name, + get_code_filename(cur_frame.f_code), + cur_frame.f_code.co_argcount, + vars, + FRAME_KIND_PYTHON, + None, + None + ) + + frames.append(frame_info) + + cur_frame = cur_frame.f_back + + return frames + + def collect_variables(self, vars, objects, names, treated, skip_unknown = False): + for name in names: + if name not in treated: + try: + obj = objects[name] + try: + if sys.version[0] == '2' and type(obj) is types.InstanceType: + type_name = "instance (" + obj.__class__.__name__ + ")" + else: + type_name = type(obj).__name__ + except: + type_name = 'unknown' + except: + if skip_unknown: + continue + obj = SynthesizedValue('', len_value=0) + type_name = 'unknown' + vars.append((name, type(obj), safe_repr(obj), safe_hex_repr(obj), type_name, get_object_len(obj))) + treated.add(name) + + def send_frame_list(self, frames, thread_name = None): + with _SendLockCtx: + write_bytes(conn, THRF) + write_int(conn, self.id) + write_string(conn, thread_name) + + # send the frame count + write_int(conn, len(frames)) + for firstlineno, lineno, curlineno, name, filename, argcount, variables, frameKind, sourceFile, sourceLine in frames: + # send each frame + write_int(conn, firstlineno) + write_int(conn, lineno) + write_int(conn, curlineno) + + write_string(conn, name) + write_string(conn, filename) + write_int(conn, argcount) + + write_int(conn, frameKind) + if frameKind == FRAME_KIND_DJANGO: + write_string(conn, sourceFile) + write_int(conn, sourceLine) + + write_int(conn, len(variables)) + for name, type_obj, safe_repr_obj, hex_repr_obj, type_name, obj_len in variables: + write_string(conn, name) + write_object(conn, type_obj, safe_repr_obj, hex_repr_obj, type_name, obj_len) + + def enum_thread_frames_locally(self): + global _threading + if _threading is None: + import threading + _threading = threading + self.send_frame_list(self.get_frame_list(), getattr(_threading.currentThread(), 'name', 'Python Thread')) + +class Module(object): + """tracks information about a loaded module""" + + CurrentLoadIndex = 0 + + def __init__(self, filename): + # TODO: Module.CurrentLoadIndex thread safety + self.module_id = Module.CurrentLoadIndex + Module.CurrentLoadIndex += 1 + self.filename = filename + +def get_code(func): + return getattr(func, 'func_code', None) or getattr(func, '__code__', None) + +class DebuggerExitException(Exception): pass + +def add_break_point(bp): + cur_bp = BREAKPOINTS.get(bp.lineno) + if cur_bp is None: + cur_bp = BREAKPOINTS[bp.lineno] = dict() + cur_bp[(bp.filename, bp.breakpoint_id)] = bp + +def try_bind_break_point(mod_filename, module, bp): + if module.filename.lower() == path.abspath(bp.filename).lower(): + bp.filename = mod_filename + bp.is_bound = True + add_break_point(bp) + report_breakpoint_bound(bp.breakpoint_id) + return True + return False + +def mark_all_threads_for_break(stepping = STEPPING_BREAK, skip_thread = None): + THREADS_LOCK.acquire() + for thread in THREADS.values(): + if thread is skip_thread: + continue + thread.stepping = stepping + THREADS_LOCK.release() + +class DebuggerLoop(object): + + instance = None + + def __init__(self, conn): + DebuggerLoop.instance = self + self.conn = conn + self.repl_backend = None + self.command_table = { + to_bytes('stpi') : self.command_step_into, + to_bytes('stpo') : self.command_step_out, + to_bytes('stpv') : self.command_step_over, + to_bytes('brkp') : self.command_set_breakpoint, + to_bytes('brkc') : self.command_set_breakpoint_condition, + to_bytes('bkpc') : self.command_set_breakpoint_pass_count, + to_bytes('bkgh') : self.command_get_breakpoint_hit_count, + to_bytes('bksh') : self.command_set_breakpoint_hit_count, + to_bytes('brkr') : self.command_remove_breakpoint, + to_bytes('brka') : self.command_break_all, + to_bytes('resa') : self.command_resume_all, + to_bytes('rest') : self.command_resume_thread, + to_bytes('ares') : self.command_auto_resume, + to_bytes('exec') : self.command_execute_code, + to_bytes('chld') : self.command_enum_children, + to_bytes('setl') : self.command_set_lineno, + to_bytes('detc') : self.command_detach, + to_bytes('clst') : self.command_clear_stepping, + to_bytes('sexi') : self.command_set_exception_info, + to_bytes('sehi') : self.command_set_exception_handler_info, + to_bytes('bkdr') : self.command_remove_django_breakpoint, + to_bytes('bkda') : self.command_add_django_breakpoint, + to_bytes('crep') : self.command_connect_repl, + to_bytes('drep') : self.command_disconnect_repl, + to_bytes('lack') : self.command_last_ack, + } + + def loop(self): + try: + while True: + inp = read_bytes(conn, 4) + cmd = self.command_table.get(inp) + if cmd is not None: + cmd() + else: + if inp: + print ('unknown command', inp) + break + except DebuggerExitException: + pass + except socket.error: + pass + except: + traceback.print_exc() + + def command_step_into(self): + tid = read_int(self.conn) + thread = get_thread_from_id(tid) + if thread is not None: + assert thread._is_blocked + thread.stepping = STEPPING_INTO + self.command_resume_all() + + def command_step_out(self): + tid = read_int(self.conn) + thread = get_thread_from_id(tid) + if thread is not None: + assert thread._is_blocked + thread.stepping = STEPPING_OUT + self.command_resume_all() + + def command_step_over(self): + # set step over + tid = read_int(self.conn) + thread = get_thread_from_id(tid) + if thread is not None: + assert thread._is_blocked + if DJANGO_DEBUG: + source_obj = get_django_frame_source(thread.cur_frame) + if source_obj is not None: + thread.django_stepping = True + self.command_resume_all() + return + + thread.stepping = STEPPING_OVER + self.command_resume_all() + + def command_set_breakpoint(self): + breakpoint_id = read_int(self.conn) + lineno = read_int(self.conn) + filename = read_string(self.conn) + condition_kind = read_int(self.conn) + condition = read_string(self.conn) + pass_count_kind = read_int(self.conn) + pass_count = read_int(self.conn) + bp = BreakpointInfo(breakpoint_id, filename, lineno, condition_kind, condition, pass_count_kind, pass_count) + + for mod_filename, module in MODULES: + if try_bind_break_point(mod_filename, module, bp): + break + else: + # Failed to bind break point (e.g. module is not loaded yet); report as pending. + add_break_point(bp) + PENDING_BREAKPOINTS.add(bp) + report_breakpoint_failed(breakpoint_id) + + def command_set_breakpoint_condition(self): + breakpoint_id = read_int(self.conn) + kind = read_int(self.conn) + condition = read_string(self.conn) + + bp = BreakpointInfo.find_by_id(breakpoint_id) + if bp is not None: + bp.condition_kind = kind + bp.condition = condition + + def command_set_breakpoint_pass_count(self): + breakpoint_id = read_int(self.conn) + kind = read_int(self.conn) + count = read_int(self.conn) + + bp = BreakpointInfo.find_by_id(breakpoint_id) + if bp is not None: + bp.pass_count_kind = kind + bp.pass_count = count + + def command_set_breakpoint_hit_count(self): + breakpoint_id = read_int(self.conn) + count = read_int(self.conn) + + bp = BreakpointInfo.find_by_id(breakpoint_id) + if bp is not None: + bp.hit_count = count + + def command_get_breakpoint_hit_count(self): + req_id = read_int(self.conn) + breakpoint_id = read_int(self.conn) + + bp = BreakpointInfo.find_by_id(breakpoint_id) + count = 0 + if bp is not None: + count = bp.hit_count + + with _SendLockCtx: + write_bytes(conn, BKHC) + write_int(conn, req_id) + write_int(conn, count) + + def command_remove_breakpoint(self): + line_no = read_int(self.conn) + brkpt_id = read_int(self.conn) + cur_bp = BREAKPOINTS.get(line_no) + if cur_bp is not None: + for file, id in cur_bp: + if id == brkpt_id: + del cur_bp[file, id] + if not cur_bp: + del BREAKPOINTS[line_no] + break + + def command_remove_django_breakpoint(self): + line_no = read_int(self.conn) + brkpt_id = read_int(self.conn) + filename = read_string(self.conn) + + bp_info = DJANGO_BREAKPOINTS.get(filename.lower()) + if bp_info is not None: + bp_info.remove_breakpoint(line_no) + + def command_add_django_breakpoint(self): + brkpt_id = read_int(self.conn) + line_no = read_int(self.conn) + filename = read_string(self.conn) + bp_info = DJANGO_BREAKPOINTS.get(filename.lower()) + if bp_info is None: + DJANGO_BREAKPOINTS[filename.lower()] = bp_info = DjangoBreakpointInfo(filename) + + bp_info.add_breakpoint(line_no, brkpt_id) + + def command_connect_repl(self): + port_num = read_int(self.conn) + _start_new_thread(self.connect_to_repl_backend, (port_num,)) + + def connect_to_repl_backend(self, port_num): + DONT_DEBUG.append(path.normcase(_vspr.__file__)) + self.repl_backend = _vspr.DebugReplBackend(self) + self.repl_backend.connect_from_debugger(port_num) + self.repl_backend.execution_loop() + + def connect_to_repl_backend_using_socket(self, sock): + DONT_DEBUG.append(path.normcase(_vspr.__file__)) + self.repl_backend = _vspr.DebugReplBackend(self) + self.repl_backend.connect_from_debugger_using_socket(sock) + self.repl_backend.execution_loop() + + def command_disconnect_repl(self): + if self.repl_backend is not None: + self.repl_backend.disconnect_from_debugger() + self.repl_backend = None + + def command_break_all(self): + global SEND_BREAK_COMPLETE + SEND_BREAK_COMPLETE = True + mark_all_threads_for_break() + + def command_resume_all(self): + # resume all + THREADS_LOCK.acquire() + all_threads = list(THREADS.values()) + THREADS_LOCK.release() + for thread in all_threads: + thread._block_starting_lock.acquire() + if thread.stepping == STEPPING_BREAK or thread.stepping == STEPPING_ATTACH_BREAK: + thread.stepping = STEPPING_NONE + if thread._is_blocked: + thread.unblock() + thread._block_starting_lock.release() + + def command_resume_thread(self): + tid = read_int(self.conn) + THREADS_LOCK.acquire() + thread = THREADS[tid] + THREADS_LOCK.release() + + if thread.reported_process_loaded: + thread.reported_process_loaded = False + self.command_resume_all() + else: + thread.unblock() + + def command_auto_resume(self): + tid = read_int(self.conn) + THREADS_LOCK.acquire() + thread = THREADS[tid] + THREADS_LOCK.release() + + stepping = thread.stepping + if ((stepping == STEPPING_OVER or stepping == STEPPING_INTO) and thread.cur_frame.f_lineno != thread.stopped_on_line): + report_step_finished(tid) + else: + self.command_resume_all() + + def command_set_exception_info(self): + BREAK_ON.clear() + BREAK_ON.default_mode = read_int(self.conn) + + break_on_count = read_int(self.conn) + for i in xrange(break_on_count): + mode = read_int(self.conn) + name = read_string(self.conn) + BREAK_ON.add_exception(name, mode) + + def command_set_exception_handler_info(self): + try: + filename = read_string(self.conn) + + statement_count = read_int(self.conn) + handlers = [] + for _ in xrange(statement_count): + line_start, line_end = read_int(self.conn), read_int(self.conn) + + expressions = set() + text = read_string(self.conn).strip() + while text != '-': + expressions.add(text) + text = read_string(self.conn) + + if not expressions: + expressions = set('*') + + handlers.append((line_start, line_end, expressions)) + + BREAK_ON.handler_cache[filename] = handlers + finally: + BREAK_ON.handler_lock.release() + + def command_clear_stepping(self): + tid = read_int(self.conn) + + thread = get_thread_from_id(tid) + if thread is not None: + thread.stepping = STEPPING_NONE + + def command_set_lineno(self): + tid = read_int(self.conn) + fid = read_int(self.conn) + lineno = read_int(self.conn) + try: + THREADS_LOCK.acquire() + THREADS[tid].cur_frame.f_lineno = lineno + newline = THREADS[tid].cur_frame.f_lineno + THREADS_LOCK.release() + with _SendLockCtx: + write_bytes(self.conn, SETL) + write_int(self.conn, 1) + write_int(self.conn, tid) + write_int(self.conn, newline) + except: + with _SendLockCtx: + write_bytes(self.conn, SETL) + write_int(self.conn, 0) + write_int(self.conn, tid) + write_int(self.conn, 0) + + def command_execute_code(self): + # execute given text in specified frame + text = read_string(self.conn) + tid = read_int(self.conn) # thread id + fid = read_int(self.conn) # frame id + eid = read_int(self.conn) # execution id + frame_kind = read_int(self.conn) + repr_kind = read_int(self.conn) + + thread, cur_frame = self.get_thread_and_frame(tid, fid, frame_kind) + if thread is not None and cur_frame is not None: + thread.run_on_thread(text, cur_frame, eid, frame_kind, repr_kind) + + def execute_code_no_report(self, text, tid, fid, frame_kind): + # execute given text in specified frame, without sending back the results + thread, cur_frame = self.get_thread_and_frame(tid, fid, frame_kind) + if thread is not None and cur_frame is not None: + thread.run_locally_no_report(text, cur_frame, frame_kind) + + def command_enum_children(self): + # execute given text in specified frame + text = read_string(self.conn) + tid = read_int(self.conn) # thread id + fid = read_int(self.conn) # frame id + eid = read_int(self.conn) # execution id + frame_kind = read_int(self.conn) # frame kind + + thread, cur_frame = self.get_thread_and_frame(tid, fid, frame_kind) + if thread is not None and cur_frame is not None: + thread.enum_child_on_thread(text, cur_frame, eid, frame_kind) + + def get_thread_and_frame(self, tid, fid, frame_kind): + thread = get_thread_from_id(tid) + cur_frame = None + + if thread is not None: + cur_frame = thread.cur_frame + for i in xrange(fid): + cur_frame = cur_frame.f_back + + return thread, cur_frame + + def command_detach(self): + detach_threads() + + # unload debugger DLL + global debugger_dll_handle + if debugger_dll_handle is not None: + k32 = ctypes.WinDLL('kernel32') + k32.FreeLibrary.argtypes = [ctypes.c_void_p] + k32.FreeLibrary(debugger_dll_handle) + debugger_dll_handle = None + + with _SendLockCtx: + write_bytes(conn, DETC) + detach_process() + + for callback in DETACH_CALLBACKS: + callback() + + raise DebuggerExitException() + + def command_last_ack(self): + last_ack_event.set() + +DETACH_CALLBACKS = [] + +def new_thread_wrapper(func, posargs, kwargs): + cur_thread = new_thread() + try: + sys.settrace(cur_thread.trace_func) + func(*posargs, **kwargs) + finally: + THREADS_LOCK.acquire() + if not cur_thread.detach: + del THREADS[cur_thread.id] + THREADS_LOCK.release() + + if not DETACHED: + report_thread_exit(cur_thread) + +def report_new_thread(new_thread): + ident = new_thread.id + with _SendLockCtx: + write_bytes(conn, NEWT) + write_int(conn, ident) + +def report_all_threads(): + THREADS_LOCK.acquire() + all_threads = list(THREADS.values()) + THREADS_LOCK.release() + for cur_thread in all_threads: + report_new_thread(cur_thread) + +def report_thread_exit(old_thread): + ident = old_thread.id + with _SendLockCtx: + write_bytes(conn, EXTT) + write_int(conn, ident) + +def report_exception(frame, exc_info, tid, break_type): + exc_type = exc_info[0] + exc_name = get_exception_name(exc_type) + exc_value = exc_info[1] + tb_value = exc_info[2] + + if type(exc_value) is tuple: + # exception object hasn't been created yet, create it now + # so we can get the correct msg. + exc_value = exc_type(*exc_value) + + excp_text = str(exc_value) + + with _SendLockCtx: + write_bytes(conn, EXCP) + write_string(conn, exc_name) + write_int(conn, tid) + write_int(conn, break_type) + write_string(conn, excp_text) + +def new_module(frame): + mod = Module(get_code_filename(frame.f_code)) + MODULES.append((frame.f_code.co_filename, mod)) + + return frame.f_code, mod + +def report_module_load(mod): + with _SendLockCtx: + write_bytes(conn, MODL) + write_int(conn, mod.module_id) + write_string(conn, mod.filename) + +def report_step_finished(tid): + with _SendLockCtx: + write_bytes(conn, STPD) + write_int(conn, tid) + +def report_breakpoint_bound(id): + with _SendLockCtx: + write_bytes(conn, BRKS) + write_int(conn, id) + +def report_breakpoint_failed(id): + with _SendLockCtx: + write_bytes(conn, BRKF) + write_int(conn, id) + +def report_breakpoint_hit(id, tid): + with _SendLockCtx: + write_bytes(conn, BRKH) + write_int(conn, id) + write_int(conn, tid) + +def report_process_loaded(tid): + with _SendLockCtx: + write_bytes(conn, LOAD) + write_int(conn, tid) + +def report_execution_error(exc_text, execution_id): + with _SendLockCtx: + write_bytes(conn, EXCE) + write_int(conn, execution_id) + write_string(conn, exc_text) + +def report_execution_exception(execution_id, exc_info): + try: + exc_text = str(exc_info[1]) + except: + exc_text = 'An exception was thrown' + + report_execution_error(exc_text, execution_id) + +def safe_hex_repr(obj): + try: + return hex(obj) + except: + return None + +def get_object_len(obj): + try: + return len(obj) + except: + return None + +def report_execution_result(execution_id, result, repr_kind = PYTHON_EVALUATION_RESULT_REPR_KIND_NORMAL): + if repr_kind == PYTHON_EVALUATION_RESULT_REPR_KIND_NORMAL: + flags = 0 + obj_repr = safe_repr(result) + obj_len = get_object_len(result) + hex_repr = safe_hex_repr(result) + else: + flags = PYTHON_EVALUATION_RESULT_RAW + hex_repr = None + for cls, raw_repr in TYPES_WITH_RAW_REPR.items(): + if isinstance(result, cls): + try: + obj_repr = raw_repr(result) + except: + obj_repr = None + break + obj_len = get_object_len(obj_repr) + if repr_kind == PYTHON_EVALUATION_RESULT_REPR_KIND_RAWLEN: + obj_repr = None + + res_type = type(result) + type_name = type(result).__name__ + + with _SendLockCtx: + write_bytes(conn, EXCR) + write_int(conn, execution_id) + write_object(conn, res_type, obj_repr, hex_repr, type_name, obj_len, flags) + +def report_children(execution_id, children): + children = [(name, expression, flags, safe_repr(result), safe_hex_repr(result), type(result), type(result).__name__, get_object_len(result)) for name, expression, result, flags in children] + with _SendLockCtx: + write_bytes(conn, CHLD) + write_int(conn, execution_id) + write_int(conn, len(children)) + for name, expression, flags, obj_repr, hex_repr, res_type, type_name, obj_len in children: + write_string(conn, name) + write_string(conn, expression) + write_object(conn, res_type, obj_repr, hex_repr, type_name, obj_len, flags) + +def get_code_filename(code): + return path.abspath(code.co_filename) + +NONEXPANDABLE_TYPES = [int, str, bool, float, object, type(None), unicode] +try: + NONEXPANDABLE_TYPES.append(long) +except NameError: pass + +def write_object(conn, obj_type, obj_repr, hex_repr, type_name, obj_len, flags = 0): + write_string(conn, obj_repr) + write_string(conn, hex_repr) + if obj_type is SynthesizedValue: + write_string(conn, '') + else: + write_string(conn, type_name) + if obj_type not in NONEXPANDABLE_TYPES and obj_len != 0: + flags |= PYTHON_EVALUATION_RESULT_EXPANDABLE + try: + for cls in TYPES_WITH_RAW_REPR: + if issubclass(obj_type, cls): + flags |= PYTHON_EVALUATION_RESULT_HAS_RAW_REPR + break + except: # guard against broken issubclass for types which aren't actually types, like vtkclass + pass + write_int(conn, obj_len or 0) + write_int(conn, flags) + +debugger_thread_id = -1 +_INTERCEPTING_FOR_ATTACH = False + +def intercept_threads(for_attach = False): + thread.start_new_thread = thread_creator + thread.start_new = thread_creator + + # If threading has already been imported (i.e. we're attaching), we must hot-patch threading._start_new_thread + # so that new threads started using it will be intercepted by our code. + # + # On the other hand, if threading has not been imported, we must not import it ourselves, because it will then + # treat the current thread as the main thread, which is incorrect when attaching because this code is executing + # on an ephemeral debugger attach thread that will go away shortly. We don't need to hot-patch it in that case + # anyway, because it will pick up the new thread.start_new_thread that we have set above when it's imported. + global _threading + if _threading is None and 'threading' in sys.modules: + import threading + _threading = threading + _threading._start_new_thread = thread_creator + + global _INTERCEPTING_FOR_ATTACH + _INTERCEPTING_FOR_ATTACH = for_attach + +def attach_process(port_num, debug_id, debug_options, report = False, block = False): + global conn + for i in xrange(50): + try: + conn = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + conn.connect(('127.0.0.1', port_num)) + write_string(conn, debug_id) + write_int(conn, 0) # success + break + except: + import time + time.sleep(50./1000) + else: + raise Exception('failed to attach') + attach_process_from_socket(conn, debug_options, report, block) + +def attach_process_from_socket(sock, debug_options, report = False, block = False): + global conn, attach_sent_break, DETACHED, DEBUG_STDLIB, BREAK_ON_SYSTEMEXIT_ZERO, DJANGO_DEBUG + + BREAK_ON_SYSTEMEXIT_ZERO = 'BreakOnSystemExitZero' in debug_options + DJANGO_DEBUG = 'DjangoDebugging' in debug_options + + if '' in PREFIXES: + # If one or more of the prefixes are empty, we can't reliably distinguish stdlib + # from user code, so override stdlib-only mode and allow to debug everything. + DEBUG_STDLIB = True + else: + DEBUG_STDLIB = 'DebugStdLib' in debug_options + + wait_on_normal_exit = 'WaitOnNormalExit' in debug_options + wait_on_abnormal_exit = 'WaitOnAbnormalExit' in debug_options + + def _excepthook(exc_type, exc_value, exc_tb): + # Display the exception and wait on exit + if exc_type is SystemExit: + if (wait_on_abnormal_exit and exc_value.code) or (wait_on_normal_exit and not exc_value.code): + print_exception(exc_type, exc_value, exc_tb) + do_wait() + else: + print_exception(exc_type, exc_value, exc_tb) + if wait_on_abnormal_exit: + do_wait() + sys.excepthook = sys.__excepthook__ = _excepthook + + conn = sock + attach_sent_break = False + + # start the debugging loop + global debugger_thread_id + debugger_thread_id = _start_new_thread(DebuggerLoop(conn).loop, ()) + + for mod_name, mod_value in sys.modules.items(): + try: + filename = getattr(mod_value, '__file__', None) + if filename is not None: + try: + fullpath = path.abspath(filename) + except: + pass + else: + MODULES.append((filename, Module(fullpath))) + except: + traceback.print_exc() + + if report: + THREADS_LOCK.acquire() + all_threads = list(THREADS.values()) + if block: + main_thread = THREADS[thread.get_ident()] + THREADS_LOCK.release() + for cur_thread in all_threads: + report_new_thread(cur_thread) + for filename, module in MODULES: + report_module_load(module) + DETACHED = False + + if block: + main_thread.block(lambda: report_process_loaded(thread.get_ident())) + + # intercept all new thread requests + if not _INTERCEPTING_FOR_ATTACH: + intercept_threads() + + if 'RedirectOutput' in debug_options: + enable_output_redirection() + +# Try to detach cooperatively, notifying the debugger as we do so. +def detach_process_and_notify_debugger(): + if DebuggerLoop.instance: + try: + DebuggerLoop.instance.command_detach() + except DebuggerExitException: # successfully detached + return + except: # swallow anything else, and forcibly detach below + pass + detach_process() + +def detach_process(): + global DETACHED + DETACHED = True + if not _INTERCEPTING_FOR_ATTACH: + if isinstance(sys.stdout, _DebuggerOutput): + sys.stdout = sys.stdout.old_out + if isinstance(sys.stderr, _DebuggerOutput): + sys.stderr = sys.stderr.old_out + + if not _INTERCEPTING_FOR_ATTACH: + thread.start_new_thread = _start_new_thread + thread.start_new = _start_new_thread + +def detach_threads(): + # tell all threads to stop tracing... + THREADS_LOCK.acquire() + all_threads = list(THREADS.items()) + THREADS_LOCK.release() + + for tid, pyThread in all_threads: + if not _INTERCEPTING_FOR_ATTACH: + pyThread.detach = True + pyThread.stepping = STEPPING_BREAK + + if pyThread._is_blocked: + pyThread.unblock() + + if not _INTERCEPTING_FOR_ATTACH: + THREADS_LOCK.acquire() + THREADS.clear() + THREADS_LOCK.release() + + BREAKPOINTS.clear() + +def new_thread(tid = None, set_break = False, frame = None): + # called during attach w/ a thread ID provided. + if tid == debugger_thread_id: + return None + + cur_thread = Thread(tid) + THREADS_LOCK.acquire() + THREADS[cur_thread.id] = cur_thread + THREADS_LOCK.release() + cur_thread.push_frame(frame) + if set_break: + cur_thread.stepping = STEPPING_ATTACH_BREAK + if not DETACHED: + report_new_thread(cur_thread) + return cur_thread + +def new_external_thread(): + thread = new_thread() + if not attach_sent_break: + # we are still doing the attach, make this thread break. + thread.stepping = STEPPING_ATTACH_BREAK + elif SEND_BREAK_COMPLETE: + # user requested break all, make this thread break + thread.stepping = STEPPING_BREAK + + sys.settrace(thread.trace_func) + +def do_wait(): + try: + import msvcrt + except ImportError: + sys.__stdout__.write('Press Enter to continue . . . ') + sys.__stdout__.flush() + sys.__stdin__.read(1) + else: + sys.__stdout__.write('Press any key to continue . . . ') + sys.__stdout__.flush() + msvcrt.getch() + +def enable_output_redirection(): + sys.stdout = _DebuggerOutput(sys.stdout, is_stdout = True) + sys.stderr = _DebuggerOutput(sys.stderr, is_stdout = False) + +def connect_repl_using_socket(sock): + _start_new_thread(DebuggerLoop.instance.connect_to_repl_backend_using_socket, (sock,)) + +class _DebuggerOutput(object): + """file like object which redirects output to the repl window.""" + errors = 'strict' + + def __init__(self, old_out, is_stdout): + self.is_stdout = is_stdout + self.old_out = old_out + if sys.version >= '3.' and hasattr(old_out, 'buffer'): + self.buffer = DebuggerBuffer(old_out.buffer) + + def flush(self): + if self.old_out: + self.old_out.flush() + + def writelines(self, lines): + for line in lines: + self.write(line) + + @property + def encoding(self): + return 'utf8' + + def write(self, value): + if not DETACHED: + probe_stack(3) + with _SendLockCtx: + write_bytes(conn, OUTP) + write_int(conn, thread.get_ident()) + write_string(conn, value) + if self.old_out: + self.old_out.write(value) + + def isatty(self): + return True + + def next(self): + pass + + @property + def name(self): + if self.is_stdout: + return "" + else: + return "" + + def __getattr__(self, name): + return getattr(self.old_out, name) + +class DebuggerBuffer(object): + def __init__(self, old_buffer): + self.buffer = old_buffer + + def write(self, data): + if not DETACHED: + probe_stack(3) + str_data = utf_8.decode(data)[0] + with _SendLockCtx: + write_bytes(conn, OUTP) + write_int(conn, thread.get_ident()) + write_string(conn, str_data) + self.buffer.write(data) + + def flush(self): + self.buffer.flush() + + def truncate(self, pos = None): + return self.buffer.truncate(pos) + + def tell(self): + return self.buffer.tell() + + def seek(self, pos, whence = 0): + return self.buffer.seek(pos, whence) + +def is_same_py_file(file1, file2): + """compares 2 filenames accounting for .pyc files""" + if file1.endswith('.pyc') or file1.endswith('.pyo'): + file1 = file1[:-1] + if file2.endswith('.pyc') or file2.endswith('.pyo'): + file2 = file2[:-1] + + return file1 == file2 + +def print_exception(exc_type, exc_value, exc_tb): + # remove debugger frames from the top and bottom of the traceback + tb = traceback.extract_tb(exc_tb) + for i in [0, -1]: + while tb: + frame_file = path.normcase(tb[i][0]) + if not any(is_same_py_file(frame_file, f) for f in DONT_DEBUG): + break + del tb[i] + + # print the traceback + if tb: + print('Traceback (most recent call last):') + for out in traceback.format_list(tb): + sys.stderr.write(out) + + # print the exception + for out in traceback.format_exception_only(exc_type, exc_value): + sys.stdout.write(out) + +def parse_debug_options(s): + return set([opt.strip() for opt in s.split(',')]) + +def debug(file, port_num, debug_id, debug_options, run_as = 'script'): + # remove us from modules so there's no trace of us + sys.modules['$visualstudio_py_debugger'] = sys.modules['visualstudio_py_debugger'] + __name__ = '$visualstudio_py_debugger' + del sys.modules['visualstudio_py_debugger'] + + wait_on_normal_exit = 'WaitOnNormalExit' in debug_options + + attach_process(port_num, debug_id, debug_options, report = True) + + # setup the current thread + cur_thread = new_thread() + cur_thread.stepping = STEPPING_LAUNCH_BREAK + + # start tracing on this thread + sys.settrace(cur_thread.trace_func) + + # now execute main file + globals_obj = {'__name__': '__main__'} + try: + if run_as == 'module': + exec_module(file, globals_obj) + elif run_as == 'code': + exec_code(file, '', globals_obj) + else: + exec_file(file, globals_obj) + finally: + sys.settrace(None) + THREADS_LOCK.acquire() + del THREADS[cur_thread.id] + THREADS_LOCK.release() + report_thread_exit(cur_thread) + + # Give VS debugger a chance to process commands + # by waiting for ack of "last" command + global _threading + if _threading is None: + import threading + _threading = threading + global last_ack_event + last_ack_event = _threading.Event() + with _SendLockCtx: + write_bytes(conn, LAST) + last_ack_event.wait(5) + + if wait_on_normal_exit: + do_wait() + +# Code objects for functions which are going to be at the bottom of the stack, right below the first +# stack frame for user code. When we walk the stack to determine whether to report or block on a given +# frame, hitting any of these means that we walked all the frames that we needed to look at. +DEBUG_ENTRYPOINTS = set(( + get_code(debug), + get_code(exec_file), + get_code(exec_module), + get_code(exec_code), + get_code(new_thread_wrapper) +)) diff --git a/pythonFiles/PythonTools/visualstudio_py_launcher.py b/pythonFiles/PythonTools/visualstudio_py_launcher.py new file mode 100644 index 000000000000..a9a2e2b17d21 --- /dev/null +++ b/pythonFiles/PythonTools/visualstudio_py_launcher.py @@ -0,0 +1,84 @@ +# Python Tools for Visual Studio +# Copyright(c) Microsoft Corporation +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the License); you may not use +# this file except in compliance with the License. You may obtain a copy of the +# License at http://www.apache.org/licenses/LICENSE-2.0 +# +# THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +# OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +# IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +# MERCHANTABLITY OR NON-INFRINGEMENT. +# +# See the Apache Version 2.0 License for specific language governing +# permissions and limitations under the License. + +""" +Starts Debugging, expected to start with normal program +to start as first argument and directory to run from as +the second argument. +""" + +__author__ = "Microsoft Corporation " +__version__ = "3.0.0.0" + +import os +import os.path +import sys +import traceback +try: + import visualstudio_py_debugger as vspd +except: + traceback.print_exc() + print(''' +Internal error detected. Please copy the above traceback and report at +http://go.microsoft.com/fwlink/?LinkId=293415 + +Press Enter to close. . .''') + try: + raw_input() + except NameError: + input() + sys.exit(1) + +# Arguments are: +# 1. Working directory. +# 2. VS debugger port to connect to. +# 3. GUID for the debug session. +# 4. Debug options (as integer - see enum PythonDebugOptions). +# 5. '-m' or '-c' to override the default run-as mode. [optional] +# 6. Startup script name. +# 7. Script arguments. + +# change to directory we expected to start from +os.chdir(sys.argv[1]) + +port_num = int(sys.argv[2]) +debug_id = sys.argv[3] +debug_options = vspd.parse_debug_options(sys.argv[4]) +del sys.argv[0:5] + +# set run_as mode appropriately +run_as = 'script' +if sys.argv and sys.argv[0] == '-m': + run_as = 'module' + del sys.argv[0] +if sys.argv and sys.argv[0] == '-c': + run_as = 'code' + del sys.argv[0] + +# preserve filename before we del sys +filename = sys.argv[0] + +# fix sys.path to be the script file dir +sys.path[0] = '' + +# exclude ourselves from being debugged +vspd.DONT_DEBUG.append(os.path.normcase(__file__)) + +# remove all state we imported +del sys, os + +# and start debugging +vspd.debug(filename, port_num, debug_id, debug_options, run_as) diff --git a/pythonFiles/PythonTools/visualstudio_py_repl.py b/pythonFiles/PythonTools/visualstudio_py_repl.py new file mode 100644 index 000000000000..4f7529fec542 --- /dev/null +++ b/pythonFiles/PythonTools/visualstudio_py_repl.py @@ -0,0 +1,1381 @@ +# Python Tools for Visual Studio +# Copyright(c) Microsoft Corporation +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the License); you may not use +# this file except in compliance with the License. You may obtain a copy of the +# License at http://www.apache.org/licenses/LICENSE-2.0 +# +# THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +# OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +# IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +# MERCHANTABLITY OR NON-INFRINGEMENT. +# +# See the Apache Version 2.0 License for specific language governing +# permissions and limitations under the License. + +from __future__ import with_statement + +__author__ = "Microsoft Corporation " +__version__ = "3.0.0.0" + +# This module MUST NOT import threading in global scope. This is because in a direct (non-ptvsd) +# attach scenario, it is loaded on the injected debugger attach thread, and if threading module +# hasn't been loaded already, it will assume that the thread on which it is being loaded is the +# main thread. This will cause issues when the thread goes away after attach completes. + +try: + import thread +except ImportError: + # Renamed in Python3k + import _thread as thread +try: + from ssl import SSLError +except: + SSLError = None + +import sys +import socket +import select +import time +import struct +import imp +import traceback +import random +import os +import inspect +import types +from collections import deque + +try: + # In the local attach scenario, visualstudio_py_util is injected into globals() + # by PyDebugAttach before loading this module, and cannot be imported. + _vspu = visualstudio_py_util +except: + try: + import visualstudio_py_util as _vspu + except ImportError: + import ptvsd.visualstudio_py_util as _vspu +to_bytes = _vspu.to_bytes +read_bytes = _vspu.read_bytes +read_int = _vspu.read_int +read_string = _vspu.read_string +write_bytes = _vspu.write_bytes +write_int = _vspu.write_int +write_string = _vspu.write_string + +try: + unicode +except NameError: + unicode = str + +try: + BaseException +except NameError: + # BaseException not defined until Python 2.5 + BaseException = Exception + +DEBUG = os.environ.get('DEBUG_REPL') is not None + +__all__ = ['ReplBackend', 'BasicReplBackend', 'BACKEND'] + +def _debug_write(out): + if DEBUG: + sys.__stdout__.write(out) + sys.__stdout__.flush() + + +class SafeSendLock(object): + """a lock which ensures we're released if we take a KeyboardInterrupt exception acquiring it""" + def __init__(self): + self.lock = thread.allocate_lock() + + def __enter__(self): + self.acquire() + + def __exit__(self, exc_type, exc_value, tb): + self.release() + + def acquire(self): + try: + self.lock.acquire() + except KeyboardInterrupt: + try: + self.lock.release() + except: + pass + raise + + def release(self): + self.lock.release() + +def _command_line_to_args_list(cmdline): + """splits a string into a list using Windows command line syntax.""" + args_list = [] + + if cmdline and cmdline.strip(): + from ctypes import c_int, c_voidp, c_wchar_p + from ctypes import byref, POINTER, WinDLL + + clta = WinDLL('shell32').CommandLineToArgvW + clta.argtypes = [c_wchar_p, POINTER(c_int)] + clta.restype = POINTER(c_wchar_p) + + lf = WinDLL('kernel32').LocalFree + lf.argtypes = [c_voidp] + + pNumArgs = c_int() + r = clta(cmdline, byref(pNumArgs)) + if r: + for index in range(0, pNumArgs.value): + if sys.hexversion >= 0x030000F0: + argval = r[index] + else: + argval = r[index].encode('ascii', 'replace') + args_list.append(argval) + lf(r) + else: + sys.stderr.write('Error parsing script arguments:\n') + sys.stderr.write(cmdline + '\n') + + return args_list + + +class UnsupportedReplException(Exception): + def __init__(self, reason): + self.reason = reason + +# save the start_new_thread so we won't debug/break into the REPL comm thread. +start_new_thread = thread.start_new_thread +class ReplBackend(object): + """back end for executing REPL code. This base class handles all of the +communication with the remote process while derived classes implement the +actual inspection and introspection.""" + _MRES = to_bytes('MRES') + _SRES = to_bytes('SRES') + _MODS = to_bytes('MODS') + _IMGD = to_bytes('IMGD') + _PRPC = to_bytes('PRPC') + _RDLN = to_bytes('RDLN') + _STDO = to_bytes('STDO') + _STDE = to_bytes('STDE') + _DBGA = to_bytes('DBGA') + _DETC = to_bytes('DETC') + _DPNG = to_bytes('DPNG') + _DXAM = to_bytes('DXAM') + + _MERR = to_bytes('MERR') + _SERR = to_bytes('SERR') + _ERRE = to_bytes('ERRE') + _EXIT = to_bytes('EXIT') + _DONE = to_bytes('DONE') + _MODC = to_bytes('MODC') + + def __init__(self): + import threading + self.conn = None + self.send_lock = SafeSendLock() + self.input_event = threading.Lock() + self.input_event.acquire() # lock starts acquired (we use it like a manual reset event) + self.input_string = None + self.exit_requested = False + + def connect(self, port): + self.conn = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self.conn.connect(('127.0.0.1', port)) + + # start a new thread for communicating w/ the remote process + start_new_thread(self._repl_loop, ()) + + def connect_using_socket(self, socket): + self.conn = socket + start_new_thread(self._repl_loop, ()) + + def _repl_loop(self): + """loop on created thread which processes communicates with the REPL window""" + try: + while True: + if self.check_for_exit_repl_loop(): + break + + # we receive a series of 4 byte commands. Each command then + # has it's own format which we must parse before continuing to + # the next command. + self.flush() + self.conn.settimeout(10) + + # 2.x raises SSLError in case of timeout (http://bugs.python.org/issue10272) + if SSLError: + timeout_exc_types = (socket.timeout, SSLError) + else: + timeout_exc_types = socket.timeout + try: + inp = read_bytes(self.conn, 4) + except timeout_exc_types: + r, w, x = select.select([], [], [self.conn], 0) + if x: + # an exception event has occured on the socket... + raise + continue + + self.conn.settimeout(None) + if inp == '': + break + self.flush() + + cmd = ReplBackend._COMMANDS.get(inp) + if cmd is not None: + cmd(self) + except: + _debug_write('error in repl loop') + _debug_write(traceback.format_exc()) + self.exit_process() + + time.sleep(2) # try and exit gracefully, then interrupt main if necessary + + if sys.platform == 'cli': + # just kill us as fast as possible + import System + System.Environment.Exit(1) + + self.interrupt_main() + + def check_for_exit_repl_loop(self): + return False + + def _cmd_run(self): + """runs the received snippet of code""" + self.run_command(read_string(self.conn)) + + def _cmd_abrt(self): + """aborts the current running command""" + # abort command, interrupts execution of the main thread. + self.interrupt_main() + + def _cmd_exit(self): + """exits the interactive process""" + self.exit_requested = True + self.exit_process() + + def _cmd_mems(self): + """gets the list of members available for the given expression""" + expression = read_string(self.conn) + try: + name, inst_members, type_members = self.get_members(expression) + except: + with self.send_lock: + write_bytes(self.conn, ReplBackend._MERR) + _debug_write('error in eval') + _debug_write(traceback.format_exc()) + else: + with self.send_lock: + write_bytes(self.conn, ReplBackend._MRES) + write_string(self.conn, name) + self._write_member_dict(inst_members) + self._write_member_dict(type_members) + + def _cmd_sigs(self): + """gets the signatures for the given expression""" + expression = read_string(self.conn) + try: + sigs = self.get_signatures(expression) + except: + with self.send_lock: + write_bytes(self.conn, ReplBackend._SERR) + _debug_write('error in eval') + _debug_write(traceback.format_exc()) + else: + with self.send_lock: + write_bytes(self.conn, ReplBackend._SRES) + # single overload + write_int(self.conn, len(sigs)) + for doc, args, vargs, varkw, defaults in sigs: + # write overload + write_string(self.conn, (doc or '')[:4096]) + arg_count = len(args) + (vargs is not None) + (varkw is not None) + write_int(self.conn, arg_count) + + def_values = [''] * (len(args) - len(defaults)) + ['=' + d for d in defaults] + for arg, def_value in zip(args, def_values): + write_string(self.conn, (arg or '') + def_value) + if vargs is not None: + write_string(self.conn, '*' + vargs) + if varkw is not None: + write_string(self.conn, '**' + varkw) + + def _cmd_setm(self): + global exec_mod + """sets the current module which code will execute against""" + mod_name = read_string(self.conn) + self.set_current_module(mod_name) + + def _cmd_sett(self): + """sets the current thread and frame which code will execute against""" + thread_id = read_int(self.conn) + frame_id = read_int(self.conn) + frame_kind = read_int(self.conn) + self.set_current_thread_and_frame(thread_id, frame_id, frame_kind) + + def _cmd_mods(self): + """gets the list of available modules""" + try: + res = self.get_module_names() + res.sort() + except: + res = [] + + with self.send_lock: + write_bytes(self.conn, ReplBackend._MODS) + write_int(self.conn, len(res)) + for name, filename in res: + write_string(self.conn, name) + write_string(self.conn, filename) + + def _cmd_inpl(self): + """handles the input command which returns a string of input""" + self.input_string = read_string(self.conn) + self.input_event.release() + + def _cmd_excf(self): + """handles executing a single file""" + filename = read_string(self.conn) + args = read_string(self.conn) + self.execute_file(filename, args) + + def _cmd_excx(self): + """handles executing a single file, module or process""" + filetype = read_string(self.conn) + filename = read_string(self.conn) + args = read_string(self.conn) + self.execute_file_ex(filetype, filename, args) + + def _cmd_debug_attach(self): + import visualstudio_py_debugger + port = read_int(self.conn) + id = read_string(self.conn) + debug_options = visualstudio_py_debugger.parse_debug_options(read_string(self.conn)) + self.attach_process(port, id, debug_options) + + _COMMANDS = { + to_bytes('run '): _cmd_run, + to_bytes('abrt'): _cmd_abrt, + to_bytes('exit'): _cmd_exit, + to_bytes('mems'): _cmd_mems, + to_bytes('sigs'): _cmd_sigs, + to_bytes('mods'): _cmd_mods, + to_bytes('setm'): _cmd_setm, + to_bytes('sett'): _cmd_sett, + to_bytes('inpl'): _cmd_inpl, + to_bytes('excf'): _cmd_excf, + to_bytes('excx'): _cmd_excx, + to_bytes('dbga'): _cmd_debug_attach, + } + + def _write_member_dict(self, mem_dict): + write_int(self.conn, len(mem_dict)) + for name, type_name in mem_dict.items(): + write_string(self.conn, name) + write_string(self.conn, type_name) + + def on_debugger_detach(self): + with self.send_lock: + write_bytes(self.conn, ReplBackend._DETC) + + def init_debugger(self): + from os import path + sys.path.append(path.dirname(__file__)) + import visualstudio_py_debugger + visualstudio_py_debugger.DONT_DEBUG.append(path.normcase(__file__)) + new_thread = visualstudio_py_debugger.new_thread() + sys.settrace(new_thread.trace_func) + visualstudio_py_debugger.intercept_threads(True) + + def send_image(self, filename): + with self.send_lock: + write_bytes(self.conn, ReplBackend._IMGD) + write_string(self.conn, filename) + + def write_png(self, image_bytes): + with self.send_lock: + write_bytes(self.conn, ReplBackend._DPNG) + write_int(self.conn, len(image_bytes)) + write_bytes(self.conn, image_bytes) + + def write_xaml(self, xaml_bytes): + with self.send_lock: + write_bytes(self.conn, ReplBackend._DXAM) + write_int(self.conn, len(xaml_bytes)) + write_bytes(self.conn, xaml_bytes) + + def send_prompt(self, ps1, ps2, update_all = True): + """sends the current prompt to the interactive window""" + with self.send_lock: + write_bytes(self.conn, ReplBackend._PRPC) + write_string(self.conn, ps1) + write_string(self.conn, ps2) + write_int(self.conn, update_all) + + def send_error(self): + """reports that an error occured to the interactive window""" + with self.send_lock: + write_bytes(self.conn, ReplBackend._ERRE) + + def send_exit(self): + """reports the that the REPL process has exited to the interactive window""" + with self.send_lock: + write_bytes(self.conn, ReplBackend._EXIT) + + def send_command_executed(self): + with self.send_lock: + write_bytes(self.conn, ReplBackend._DONE) + + def send_modules_changed(self): + with self.send_lock: + write_bytes(self.conn, ReplBackend._MODC) + + def read_line(self): + """reads a line of input from standard input""" + with self.send_lock: + write_bytes(self.conn, ReplBackend._RDLN) + self.input_event.acquire() + return self.input_string + + def write_stdout(self, value): + """writes a string to standard output in the remote console""" + with self.send_lock: + write_bytes(self.conn, ReplBackend._STDO) + write_string(self.conn, value) + + def write_stderr(self, value): + """writes a string to standard input in the remote console""" + with self.send_lock: + write_bytes(self.conn, ReplBackend._STDE) + write_string(self.conn, value) + + ################################################################ + # Implementation of execution, etc... + + def execution_loop(self): + """starts processing execution requests""" + raise NotImplementedError + + def run_command(self, command): + """runs the specified command which is a string containing code""" + raise NotImplementedError + + def execute_file(self, filename, args): + """executes the given filename as the main module""" + return self.execute_file_ex('script', filename, args) + + def execute_file_ex(self, filetype, filename, args): + """executes the given filename as a 'script', 'module' or 'process'.""" + raise NotImplementedError + + def interrupt_main(self): + """aborts the current running command""" + raise NotImplementedError + + def exit_process(self): + """exits the REPL process""" + raise NotImplementedError + + def get_members(self, expression): + """returns a tuple of the type name, instance members, and type members""" + raise NotImplementedError + + def get_signatures(self, expression): + """returns doc, args, vargs, varkw, defaults.""" + raise NotImplementedError + + def set_current_module(self, module): + """sets the module which code executes against""" + raise NotImplementedError + + def set_current_thread_and_frame(self, thread_id, frame_id, frame_kind): + """sets the current thread and frame which code will execute against""" + raise NotImplementedError + + def get_module_names(self): + """returns a list of module names""" + raise NotImplementedError + + def flush(self): + """flushes the stdout/stderr buffers""" + raise NotImplementedError + + def attach_process(self, port, debugger_id, debug_options): + """starts processing execution requests""" + raise NotImplementedError + +def exit_work_item(): + sys.exit(0) + + +if sys.platform == 'cli': + # We need special handling to reset the abort for keyboard interrupt exceptions + class ReplAbortException(Exception): pass + + import clr + clr.AddReference('Microsoft.Dynamic') + clr.AddReference('Microsoft.Scripting') + clr.AddReference('IronPython') + from Microsoft.Scripting import KeyboardInterruptException + from Microsoft.Scripting import ParamDictionaryAttribute + from IronPython.Runtime.Operations import PythonOps + from IronPython.Runtime import PythonContext + from Microsoft.Scripting import SourceUnit, SourceCodeKind + from Microsoft.Scripting.Runtime import Scope + + python_context = clr.GetCurrentRuntime().GetLanguage(PythonContext) + + from System import DBNull, ParamArrayAttribute + builtin_method_descriptor_type = type(list.append) + + import System + NamespaceType = type(System) + +class _OldClass: + pass + +_OldClassType = type(_OldClass) +_OldInstanceType = type(_OldClass()) + +class BasicReplBackend(ReplBackend): + future_bits = 0x3e010 # code flags used to mark future bits + + """Basic back end which executes all Python code in-proc""" + def __init__(self, mod_name = '__main__', launch_file = None): + import threading + ReplBackend.__init__(self) + if mod_name is not None: + if sys.platform == 'cli': + self.exec_mod = Scope() + self.exec_mod.__name__ = '__main__' + else: + sys.modules[mod_name] = self.exec_mod = imp.new_module(mod_name) + else: + self.exec_mod = sys.modules['__main__'] + + self.launch_file = launch_file + self.code_flags = 0 + self.execute_item = None + self.execute_item_lock = threading.Lock() + self.execute_item_lock.acquire() # lock starts acquired (we use it like manual reset event) + + def init_connection(self): + sys.stdout = _ReplOutput(self, is_stdout = True) + sys.stderr = _ReplOutput(self, is_stdout = False) + sys.stdin = _ReplInput(self) + if sys.platform == 'cli': + import System + System.Console.SetOut(DotNetOutput(self, True)) + System.Console.SetError(DotNetOutput(self, False)) + + def connect(self, port): + ReplBackend.connect(self, port) + self.init_connection() + + def connect_using_socket(self, socket): + ReplBackend.connect_using_socket(self, socket) + self.init_connection() + + + def run_file_as_main(self, filename, args): + f = open(filename, 'rb') + try: + contents = f.read().replace(to_bytes('\r\n'), to_bytes('\n')) + finally: + f.close() + sys.argv = [filename] + sys.argv.extend(_command_line_to_args_list(args)) + self.exec_mod.__file__ = filename + if sys.platform == 'cli': + code = python_context.CreateSnippet(contents, None, SourceCodeKind.File) + code.Execute(self.exec_mod) + else: + self.code_flags = 0 + real_file = filename + if isinstance(filename, unicode) and unicode is not str: + # http://pytools.codeplex.com/workitem/696 + # We need to encode the unicode filename here, Python 2.x will throw trying + # to convert it to ASCII instead of the filesystem encoding. + real_file = filename.encode(sys.getfilesystemencoding()) + code = compile(contents, real_file, 'exec') + self.code_flags |= (code.co_flags & BasicReplBackend.future_bits) + exec(code, self.exec_mod.__dict__, self.exec_mod.__dict__) + + def python_executor(self, code): + """we can't close over unbound variables in execute_code_work_item +due to the exec, so we do it here""" + def func(): + code.Execute(self.exec_mod) + return func + + def execute_code_work_item(self): + _debug_write('Executing: ' + repr(self.current_code)) + stripped_code = self.current_code.strip() + + if sys.platform == 'cli': + code_to_send = '' + for line in stripped_code.split('\n'): + stripped = line.strip() + if (stripped.startswith('#') or not stripped) and not code_to_send: + continue + code_to_send += line + '\n' + + code = python_context.CreateSnippet(code_to_send, None, SourceCodeKind.InteractiveCode) + dispatcher = clr.GetCurrentRuntime().GetLanguage(PythonContext).GetCommandDispatcher() + if dispatcher is not None: + dispatcher(self.python_executor(code)) + else: + code.Execute(self.exec_mod) + else: + code = compile(self.current_code, '', 'single', self.code_flags) + self.code_flags |= (code.co_flags & BasicReplBackend.future_bits) + exec(code, self.exec_mod.__dict__, self.exec_mod.__dict__) + self.current_code = None + + def run_one_command(self, cur_modules, cur_ps1, cur_ps2): + # runs a single iteration of an input, execute file, etc... + # This is extracted into it's own method so we play nice w/ IronPython thread abort. + # Otherwise we have a nested exception hanging around and the 2nd abort doesn't + # work (that's probably an IronPython bug) + try: + new_modules = self._get_cur_module_set() + try: + if new_modules != cur_modules: + self.send_modules_changed() + except: + pass + cur_modules = new_modules + + self.execute_item_lock.acquire() + + if self.check_for_exit_execution_loop(): + return True, None, None, None + + if self.execute_item is not None: + try: + self.execute_item() + finally: + self.execute_item = None + + try: + self.send_command_executed() + except SocketError: + return True, None, None, None + + try: + if cur_ps1 != sys.ps1 or cur_ps2 != sys.ps2: + new_ps1 = str(sys.ps1) + new_ps2 = str(sys.ps2) + + self.send_prompt(new_ps1, new_ps2) + + cur_ps1 = new_ps1 + cur_ps2 = new_ps2 + except: + pass + except SystemExit: + self.send_error() + self.send_exit() + # wait for ReplEvaluator to send back exit requested which will indicate + # that all the output has been processed. + while not self.exit_requested: + time.sleep(.25) + return True, None, None, None + except BaseException: + _debug_write('Exception') + exc_type, exc_value, exc_tb = sys.exc_info() + if sys.platform == 'cli': + if isinstance(exc_value.clsException, System.Threading.ThreadAbortException): + try: + System.Threading.Thread.ResetAbort() + except SystemError: + pass + sys.stderr.write('KeyboardInterrupt') + else: + # let IronPython format the exception so users can do -X:ExceptionDetail or -X:ShowClrExceptions + exc_next = self.skip_internal_frames(exc_tb) + sys.stderr.write(''.join(traceback.format_exception(exc_type, exc_value, exc_next))) + else: + exc_next = self.skip_internal_frames(exc_tb) + sys.stderr.write(''.join(traceback.format_exception(exc_type, exc_value, exc_next))) + + try: + self.send_error() + except SocketError: + _debug_write('err sending DONE') + return True, None, None, None + + return False, cur_modules, cur_ps1, cur_ps2 + + def skip_internal_frames(self, tb): + """return the first frame outside of the repl/debugger code""" + while tb is not None and self.is_internal_frame(tb): + tb = tb.tb_next + return tb + + def is_internal_frame(self, tb): + """return true if the frame is from internal code (repl or debugger)""" + f = tb.tb_frame + co = f.f_code + filename = co.co_filename + return filename.endswith('visualstudio_py_repl.py') or filename.endswith('visualstudio_py_debugger.py') + + def execution_loop(self): + """loop on the main thread which is responsible for executing code""" + + if sys.platform == 'cli' and sys.version_info[:3] < (2, 7, 1): + # IronPython doesn't support thread.interrupt_main until 2.7.1 + import System + self.main_thread = System.Threading.Thread.CurrentThread + + # save our selves so global lookups continue to work (required pre-2.6)... + cur_modules = set() + try: + cur_ps1 = sys.ps1 + cur_ps2 = sys.ps2 + except: + # CPython/IronPython don't set sys.ps1 for non-interactive sessions, Jython and PyPy do + sys.ps1 = cur_ps1 = '>>> ' + sys.ps2 = cur_ps2 = '... ' + + self.send_prompt(cur_ps1, cur_ps2) + + # launch the startup script if one has been specified + if self.launch_file: + try: + self.run_file_as_main(self.launch_file, '') + except: + print('error in launching startup script:') + traceback.print_exc() + + while True: + exit, cur_modules, cur_ps1, cur_ps2 = self.run_one_command(cur_modules, cur_ps1, cur_ps2) + if exit: + return + + def check_for_exit_execution_loop(self): + return False + + def execute_script_work_item(self): + self.run_file_as_main(self.current_code, self.current_args) + + def execute_module_work_item(self): + new_argv = [''] + _command_line_to_args_list(self.current_args) + old_argv = sys.argv + import runpy + try: + sys.argv = new_argv + runpy.run_module(self.current_code, alter_sys=True) + except Exception: + traceback.print_exc() + finally: + sys.argv = old_argv + + def execute_process_work_item(self): + try: + from subprocess import Popen, PIPE, STDOUT + import codecs + out_codec = codecs.lookup(sys.stdout.encoding) + + proc = Popen( + '"%s" %s' % (self.current_code, self.current_args), + stdout=PIPE, + stderr=STDOUT, + bufsize=0, + ) + + for line in proc.stdout: + print(out_codec.decode(line, 'replace')[0].rstrip('\r\n')) + except Exception: + traceback.print_exc() + + @staticmethod + def _get_cur_module_set(): + """gets the set of modules avoiding exceptions if someone puts something + weird in there""" + + try: + return set(sys.modules) + except: + res = set() + for name in sys.modules: + try: + res.add(name) + except: + pass + return res + + + def run_command(self, command): + self.current_code = command + self.execute_item = self.execute_code_work_item + self.execute_item_lock.release() + + def execute_file_ex(self, filetype, filename, args): + self.current_code = filename + self.current_args = args + self.execute_item = getattr(self, 'execute_%s_work_item' % filetype, None) + self.execute_item_lock.release() + + def interrupt_main(self): + # acquire the send lock so we dont interrupt while we're communicting w/ the debugger + with self.send_lock: + if sys.platform == 'cli' and sys.version_info[:3] < (2, 7, 1): + # IronPython doesn't get thread.interrupt_main until 2.7.1 + self.main_thread.Abort(ReplAbortException()) + else: + thread.interrupt_main() + + def exit_process(self): + self.execute_item = exit_work_item + try: + self.execute_item_lock.release() + except: + pass + sys.exit(0) + + def get_members(self, expression): + """returns a tuple of the type name, instance members, and type members""" + getattr_func = getattr + if not expression: + all_members = {} + if sys.platform == 'cli': + code = python_context.CreateSnippet('vars()', None, SourceCodeKind.AutoDetect) + items = code.Execute(self.exec_mod) + else: + items = self.exec_mod.__dict__ + + for key, value in items.items(): + all_members[key] = self.get_type_name(value) + return '', all_members, {} + else: + if sys.platform == 'cli': + code = python_context.CreateSnippet(expression, None, SourceCodeKind.AutoDetect) + val = code.Execute(self.exec_mod) + + code = python_context.CreateSnippet('dir(' + expression + ')', None, SourceCodeKind.AutoDetect) + members = code.Execute(self.exec_mod) + + code = python_context.CreateSnippet('lambda value, name: getattr(value, name)', None, SourceCodeKind.AutoDetect) + getattr_func = code.Execute(self.exec_mod) + else: + val = eval(expression, self.exec_mod.__dict__, self.exec_mod.__dict__) + members = dir(val) + + return self.collect_members(val, members, getattr_func) + + def collect_members(self, val, members, getattr_func): + t = type(val) + + inst_members = {} + if hasattr(val, '__dict__'): + # collect the instance members + try: + for mem_name in val.__dict__: + mem_t = self._get_member_type(val, mem_name, True, getattr_func) + if mem_t is not None: + inst_members[mem_name] = mem_t + except: + pass + + # collect the type members + + type_members = {} + for mem_name in members: + if mem_name not in inst_members: + mem_t = self._get_member_type(val, mem_name, False, getattr_func) + if mem_t is not None: + type_members[mem_name] = mem_t + + + return t.__module__ + '.' + t.__name__, inst_members, type_members + + def get_ipy_sig(self, obj, ctor): + args = [] + vargs = None + varkw = None + defaults = [] + for param in ctor.GetParameters(): + if param.IsDefined(ParamArrayAttribute, False): + vargs = param.Name + elif param.IsDefined(ParamDictionaryAttribute, False): + varkw = param.Name + else: + args.append(param.Name) + + if param.DefaultValue is not DBNull.Value: + defaults.append(repr(param.DefaultValue)) + + return obj.__doc__, args, vargs, varkw, tuple(defaults) + + def get_signatures(self, expression): + if sys.platform == 'cli': + code = python_context.CreateSnippet(expression, None, SourceCodeKind.AutoDetect) + val = code.Execute(self.exec_mod) + else: + val = eval(expression, self.exec_mod.__dict__, self.exec_mod.__dict__) + + return self.collect_signatures(val) + + def collect_signatures(self, val): + doc = val.__doc__ + type_obj = None + if isinstance(val, type) or isinstance(val, _OldClassType): + type_obj = val + val = val.__init__ + + try: + args, vargs, varkw, defaults = inspect.getargspec(val) + except TypeError: + # we're not doing inspect on a Python function... + if sys.platform == 'cli': + if type_obj is not None: + clr_type = clr.GetClrType(type_obj) + ctors = clr_type.GetConstructors() + return [self.get_ipy_sig(type_obj, ctor) for ctor in ctors] + elif type(val) is types.BuiltinFunctionType: + return [self.get_ipy_sig(target, target.Targets[0]) for target in val.Overloads.Functions] + elif type(val) is builtin_method_descriptor_type: + val = PythonOps.GetBuiltinMethodDescriptorTemplate(val) + return [self.get_ipy_sig(target, target.Targets[0]) for target in val.Overloads.Functions] + raise + + remove_self = type_obj is not None or (type(val) is types.MethodType and + ((sys.version_info >= (3,) and val.__self__ is not None) or + (sys.version_info < (3,) and val.im_self is not None))) + + if remove_self: + # remove self for instance methods and types + args = args[1:] + + if defaults is not None: + defaults = [repr(default) for default in defaults] + else: + defaults = [] + return [(doc, args, vargs, varkw, defaults)] + + def set_current_module(self, module): + mod = sys.modules.get(module) + if mod is not None: + _debug_write('Setting module to ' + module) + if sys.platform == 'cli': + self.exec_mod = clr.GetClrType(type(sys)).GetProperty('Scope', System.Reflection.BindingFlags.Public | System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Instance).GetValue(sys, ()) + else: + self.exec_mod = mod + else: + _debug_write('Unknown module ' + module) + + def get_module_names(self): + res = [] + for name, module in sys.modules.items(): + try: + if name != 'visualstudio_py_repl' and name != '$visualstudio_py_debugger': + if sys.platform == 'cli' and type(module) is NamespaceType: + self.get_namespaces(name, module, res) + else: + filename = getattr(module, '__file__', '') or '' + res.append((name, filename)) + + except: + pass + return res + + def get_namespaces(self, basename, namespace, names): + names.append((basename, '')) + try: + for name in dir(namespace): + new_name = basename + '.' + name + new_namespace = getattr(namespace, name) + + if type(new_namespace) is NamespaceType: + self.get_namespaces(new_name, new_namespace, names) + except: + pass + + def flush(self): + sys.stdout.flush() + + def do_detach(self): + import visualstudio_py_debugger + visualstudio_py_debugger.DETACH_CALLBACKS.remove(self.do_detach) + self.on_debugger_detach() + + def attach_process(self, port, debugger_id, debug_options): + def execute_attach_process_work_item(): + import visualstudio_py_debugger + visualstudio_py_debugger.DETACH_CALLBACKS.append(self.do_detach) + visualstudio_py_debugger.attach_process(port, debugger_id, debug_options, report=True, block=True) + + self.execute_item = execute_attach_process_work_item + self.execute_item_lock.release() + + @staticmethod + def get_type_name(val): + try: + mem_t = type(val) + mem_t_name = mem_t.__module__ + '.' + mem_t.__name__ + return mem_t_name + except: + pass + + @staticmethod + def _get_member_type(inst, name, from_dict, getattr_func = None): + try: + if from_dict: + val = inst.__dict__[name] + elif type(inst) is _OldInstanceType: + val = getattr_func(inst.__class__, name) + else: + val = getattr_func(type(inst), name) + mem_t_name = BasicReplBackend.get_type_name(val) + return mem_t_name + except: + if not from_dict: + try: + return BasicReplBackend.get_type_name(getattr_func(inst, name)) + except: + pass + return + +class DebugReplBackend(BasicReplBackend): + def __init__(self, debugger): + BasicReplBackend.__init__(self, None, None) + self.debugger = debugger + self.thread_id = None + self.frame_id = None + self.frame_kind = None + self.disconnect_requested = False + + def init_connection(self): + sys.stdout = _ReplOutput(self, is_stdout = True, old_out = sys.stdout) + sys.stderr = _ReplOutput(self, is_stdout = False, old_out = sys.stderr) + if sys.platform == 'cli': + import System + self.old_cli_stdout = System.Console.Out + self.old_cli_stderr = System.Console.Error + System.Console.SetOut(DotNetOutput(self, True, System.Console.Out)) + System.Console.SetError(DotNetOutput(self, False, System.Console.Error)) + + def connect_from_debugger(self, port): + ReplBackend.connect(self, port) + self.init_connection() + + def connect_from_debugger_using_socket(self, socket): + ReplBackend.connect_using_socket(self, socket) + self.init_connection() + + def disconnect_from_debugger(self): + sys.stdout = sys.stdout.old_out + sys.stderr = sys.stderr.old_out + if sys.platform == 'cli': + System.Console.SetOut(self.old_cli_stdout) + System.Console.SetError(self.old_cli_stderr) + del self.old_cli_stdout + del self.old_cli_stderr + + # this tells both _repl_loop and execution_loop, each + # running on its own worker thread, to exit + self.disconnect_requested = True + self.execute_item_lock.release() + + def set_current_thread_and_frame(self, thread_id, frame_id, frame_kind): + self.thread_id = thread_id + self.frame_id = frame_id + self.frame_kind = frame_kind + self.exec_mod = None + + def execute_code_work_item(self): + if self.exec_mod is not None: + BasicReplBackend.execute_code_work_item(self) + else: + try: + self.debugger.execute_code_no_report(self.current_code, self.thread_id, self.frame_id, self.frame_kind) + finally: + self.current_code = None + + def get_members(self, expression): + """returns a tuple of the type name, instance members, and type members""" + if self.exec_mod is not None: + return BasicReplBackend.get_members(self, expression) + else: + thread, cur_frame = self.debugger.get_thread_and_frame(self.thread_id, self.frame_id, self.frame_kind) + return self.get_members_for_frame(expression, thread, cur_frame, self.frame_kind) + + def get_signatures(self, expression): + """returns doc, args, vargs, varkw, defaults.""" + if self.exec_mod is not None: + return BasicReplBackend.get_signatures(self, expression) + else: + thread, cur_frame = self.debugger.get_thread_and_frame(self.thread_id, self.frame_id, self.frame_kind) + return self.get_signatures_for_frame(expression, thread, cur_frame, self.frame_kind) + + def get_members_for_frame(self, expression, thread, cur_frame, frame_kind): + """returns a tuple of the type name, instance members, and type members""" + getattr_func = getattr + if not expression: + all_members = {} + if sys.platform == 'cli': + code = python_context.CreateSnippet('vars()', None, SourceCodeKind.AutoDetect) + globals = code.Execute(Scope(cur_frame.f_globals)) + locals = code.Execute(Scope(thread.get_locals(cur_frame, frame_kind))) + else: + globals = cur_frame.f_globals + locals = thread.get_locals(cur_frame, frame_kind) + + for key, value in globals.items(): + all_members[key] = self.get_type_name(value) + + for key, value in locals.items(): + all_members[key] = self.get_type_name(value) + + return '', all_members, {} + else: + if sys.platform == 'cli': + scope = Scope(cur_frame.f_globals) + + code = python_context.CreateSnippet(expression, None, SourceCodeKind.AutoDetect) + val = code.Execute(scope) + + code = python_context.CreateSnippet('dir(' + expression + ')', None, SourceCodeKind.AutoDetect) + members = code.Execute(scope) + + code = python_context.CreateSnippet('lambda value, name: getattr(value, name)', None, SourceCodeKind.AutoDetect) + getattr_func = code.Execute(scope) + else: + val = eval(expression, cur_frame.f_globals, thread.get_locals(cur_frame, frame_kind)) + members = dir(val) + + return self.collect_members(val, members, getattr_func) + + def get_signatures_for_frame(self, expression, thread, cur_frame, frame_kind): + if sys.platform == 'cli': + code = python_context.CreateSnippet(expression, None, SourceCodeKind.AutoDetect) + val = code.Execute(Scope(cur_frame.f_globals)) + else: + val = eval(expression, cur_frame.f_globals, thread.get_locals(cur_frame, frame_kind)) + + return self.collect_signatures(val) + + def set_current_module(self, module): + if module == '': + self.exec_mod = None + else: + BasicReplBackend.set_current_module(self, module) + + def check_for_exit_repl_loop(self): + return self.disconnect_requested + + def check_for_exit_execution_loop(self): + return self.disconnect_requested + +class _ReplOutput(object): + """file like object which redirects output to the repl window.""" + errors = None + + def __init__(self, backend, is_stdout, old_out = None): + self.name = "" if is_stdout else "" + self.backend = backend + self.old_out = old_out + self.is_stdout = is_stdout + self.pipe = None + + def flush(self): + if self.old_out: + self.old_out.flush() + + def fileno(self): + if self.pipe is None: + self.pipe = os.pipe() + thread.start_new_thread(self.pipe_thread, (), {}) + + return self.pipe[1] + + def pipe_thread(self): + while True: + data = os.read(self.pipe[0], 1) + if data == '\r': + data = os.read(self.pipe[0], 1) + if data == '\n': + self.write('\n') + else: + self.write('\r' + data) + else: + self.write(data) + + @property + def encoding(self): + return 'utf8' + + def writelines(self, lines): + for line in lines: + self.write(line) + self.write('\n') + + def write(self, value): + _debug_write('printing ' + repr(value) + '\n') + if self.is_stdout: + self.backend.write_stdout(value) + else: + self.backend.write_stderr(value) + if self.old_out: + self.old_out.write(value) + + def isatty(self): + return True + + def next(self): + pass + + +class _ReplInput(object): + """file like object which redirects input from the repl window""" + def __init__(self, backend): + self.backend = backend + + def readline(self): + return self.backend.read_line() + + def readlines(self, size = None): + res = [] + while True: + line = self.readline() + if line is not None: + res.append(line) + else: + break + + return res + + def xreadlines(self): + return self + + def write(self, *args): + raise IOError("File not open for writing") + + def flush(self): pass + + def isatty(self): + return True + + def __iter__(self): + return self + + def next(self): + return self.readline() + + +if sys.platform == 'cli': + import System + class DotNetOutput(System.IO.TextWriter): + def __new__(cls, backend, is_stdout, old_out=None): + return System.IO.TextWriter.__new__(cls) + + def __init__(self, backend, is_stdout, old_out=None): + self.backend = backend + self.is_stdout = is_stdout + self.old_out = old_out + + def Write(self, value, *args): + if self.old_out: + self.old_out.Write(value, *args) + + if not args: + if type(value) is str or type(value) is System.Char: + if self.is_stdout: + self.backend.write_stdout(str(value).replace('\r\n', '\n')) + else: + self.backend.write_stderr(str(value).replace('\r\n', '\n')) + else: + super(DotNetOutput, self).Write.Overloads[object](value) + else: + self.Write(System.String.Format(value, *args)) + + def WriteLine(self, value, *args): + if self.old_out: + self.old_out.WriteLine(value, *args) + + if not args: + if type(value) is str or type(value) is System.Char: + if self.is_stdout: + self.backend.write_stdout(str(value).replace('\r\n', '\n') + '\n') + else: + self.backend.write_stderr(str(value).replace('\r\n', '\n') + '\n') + else: + super(DotNetOutput, self).WriteLine.Overloads[object](value) + else: + self.WriteLine(System.String.Format(value, *args)) + + @property + def Encoding(self): + return System.Text.Encoding.UTF8 + + +BACKEND = None + +def _run_repl(): + from optparse import OptionParser + + parser = OptionParser(prog='repl', description='Process REPL options') + parser.add_option('--port', dest='port', + help='the port to connect back to') + parser.add_option('--launch_file', dest='launch_file', + help='the script file to run on startup') + parser.add_option('--execution_mode', dest='backend', + help='the backend to use') + parser.add_option('--enable-attach', dest='enable_attach', + action="store_true", default=False, + help='enable attaching the debugger via $attach') + + (options, args) = parser.parse_args() + + # kick off repl + # make us available under our "normal" name, not just __main__ which we'll likely replace. + sys.modules['visualstudio_py_repl'] = sys.modules['__main__'] + global __name__ + __name__ = 'visualstudio_py_repl' + + backend_type = BasicReplBackend + backend_error = None + if options.backend is not None and options.backend.lower() != 'standard': + try: + split_backend = options.backend.split('.') + backend_mod_name = '.'.join(split_backend[:-1]) + backend_name = split_backend[-1] + backend_type = getattr(__import__(backend_mod_name), backend_name) + except UnsupportedReplException: + backend_error = sys.exc_info()[1].reason + except: + backend_error = traceback.format_exc() + + # fix sys.path so that cwd is where the project lives. + sys.path[0] = '.' + # remove all of our parsed args in case we have a launch file that cares... + sys.argv = args or [''] + + global BACKEND + BACKEND = backend_type(launch_file=options.launch_file) + BACKEND.connect(int(options.port)) + + if options.enable_attach: + BACKEND.init_debugger() + + if backend_error is not None: + sys.stderr.write('Error using selected REPL back-end:\n') + sys.stderr.write(backend_error + '\n') + sys.stderr.write('Using standard backend instead\n') + + # execute code on the main thread which we can interrupt + BACKEND.execution_loop() + +if __name__ == '__main__': + try: + _run_repl() + except: + if DEBUG: + _debug_write(traceback.format_exc()) + _debug_write('exiting') + input() + raise diff --git a/pythonFiles/PythonTools/visualstudio_py_util.py b/pythonFiles/PythonTools/visualstudio_py_util.py new file mode 100644 index 000000000000..49227337b1fb --- /dev/null +++ b/pythonFiles/PythonTools/visualstudio_py_util.py @@ -0,0 +1,625 @@ +# Python Tools for Visual Studio +# Copyright(c) Microsoft Corporation +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the License); you may not use +# this file except in compliance with the License. You may obtain a copy of the +# License at http://www.apache.org/licenses/LICENSE-2.0 +# +# THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +# OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +# IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +# MERCHANTABLITY OR NON-INFRINGEMENT. +# +# See the Apache Version 2.0 License for specific language governing +# permissions and limitations under the License. + +__author__ = "Microsoft Corporation " +__version__ = "3.0.0.0" + +# This module MUST NOT import threading in global scope. This is because in a direct (non-ptvsd) +# attach scenario, it is loaded on the injected debugger attach thread, and if threading module +# hasn't been loaded already, it will assume that the thread on which it is being loaded is the +# main thread. This will cause issues when the thread goes away after attach completes. + +import imp +import os +import sys +import struct + +# Import encodings early to avoid import on the debugger thread, which may cause deadlock +from encodings import utf_8, ascii + +# WARNING: Avoid imports beyond this point, specifically on the debugger thread, as this may cause +# deadlock where the debugger thread performs an import while a user thread has the import lock + +# Py3k compat - alias unicode to str, and xrange to range +try: + unicode +except: + unicode = str +try: + xrange +except: + xrange = range + + +if sys.version_info[0] >= 3: + def to_bytes(cmd_str): + return ascii.Codec.encode(cmd_str)[0] +else: + def to_bytes(cmd_str): + return cmd_str + +def exec_code(code, file, global_variables): + '''Executes the provided code as if it were the original script provided + to python.exe. The functionality is similar to `runpy.run_path`, which was + added in Python 2.7/3.2. + + The following values in `global_variables` will be set to the following + values, if they are not already set:: + __name__ = '' + __file__ = file + __package__ = __name__.rpartition('.')[0] # 2.6 and later + __cached__ = None # 3.2 and later + __loader__ = None # 3.3 and later + + The `sys.modules` entry for ``__name__`` will be set to a new module, and + ``sys.path[0]`` will be changed to the value of `file` without the filename. + Both values are restored when this function exits. + ''' + original_main = sys.modules.get('__main__') + + global_variables = dict(global_variables) + mod_name = global_variables.setdefault('__name__', '') + mod = sys.modules[mod_name] = imp.new_module(mod_name) + mod.__dict__.update(global_variables) + global_variables = mod.__dict__ + global_variables.setdefault('__file__', file) + if sys.version_info[0] >= 3 or sys.version_info[1] >= 6: + global_variables.setdefault('__package__', mod_name.rpartition('.')[0]) + if sys.version_info[0] >= 3: + if sys.version_info[1] >= 2: + global_variables.setdefault('__cached__', None) + if sys.version_info[1] >= 3: + try: + global_variables.setdefault('__loader__', original_main.__loader__) + except AttributeError: + pass + + sys.path[0] = os.path.split(file)[0] + code_obj = compile(code, file, 'exec') + exec(code_obj, global_variables) + +def exec_file(file, global_variables): + '''Executes the provided script as if it were the original script provided + to python.exe. The functionality is similar to `runpy.run_path`, which was + added in Python 2.7/3.2. + + The following values in `global_variables` will be set to the following + values, if they are not already set:: + __name__ = '' + __file__ = file + __package__ = __name__.rpartition('.')[0] # 2.6 and later + __cached__ = None # 3.2 and later + __loader__ = sys.modules['__main__'].__loader__ # 3.3 and later + + The `sys.modules` entry for ``__name__`` will be set to a new module, and + ``sys.path[0]`` will be changed to the value of `file` without the filename. + Both values are restored when this function exits. + ''' + f = open(file, "rb") + try: + code = f.read().replace(to_bytes('\r\n'), to_bytes('\n')) + to_bytes('\n') + finally: + f.close() + exec_code(code, file, global_variables) + +def exec_module(module, global_variables): + '''Executes the provided module as if it were provided as '-m module'. The + functionality is implemented using `runpy.run_module`, which was added in + Python 2.5. + ''' + import runpy + runpy.run_module(module, global_variables, run_name=global_variables.get('__name__'), alter_sys=True) + +UNICODE_PREFIX = to_bytes('U') +ASCII_PREFIX = to_bytes('A') +NONE_PREFIX = to_bytes('N') + + +def read_bytes(conn, count): + b = to_bytes('') + while len(b) < count: + b += conn.recv(count - len(b)) + return b + + +def write_bytes(conn, b): + conn.sendall(b) + + +def read_int(conn): + return struct.unpack('!q', read_bytes(conn, 8))[0] + + +def write_int(conn, i): + write_bytes(conn, struct.pack('!q', i)) + + +def read_string(conn): + """ reads length of text to read, and then the text encoded in UTF-8, and returns the string""" + strlen = read_int(conn) + if not strlen: + return '' + res = to_bytes('') + while len(res) < strlen: + res = res + conn.recv(strlen - len(res)) + + res = utf_8.decode(res)[0] + if sys.version_info[0] == 2 and sys.platform != 'cli': + # Py 2.x, we want an ASCII string if possible + try: + res = ascii.Codec.encode(res)[0] + except UnicodeEncodeError: + pass + + return res + + +def write_string(conn, s): + if s is None: + write_bytes(conn, NONE_PREFIX) + elif isinstance(s, unicode): + b = utf_8.encode(s)[0] + b_len = len(b) + write_bytes(conn, UNICODE_PREFIX) + write_int(conn, b_len) + if b_len > 0: + write_bytes(conn, b) + else: + s_len = len(s) + write_bytes(conn, ASCII_PREFIX) + write_int(conn, s_len) + if s_len > 0: + write_bytes(conn, s) + +class SafeRepr(object): + # String types are truncated to maxstring_outer when at the outer- + # most level, and truncated to maxstring_inner characters inside + # collections. + maxstring_outer = 2 ** 16 + maxstring_inner = 30 + if sys.version_info >= (3, 0): + string_types = (str, bytes) + set_info = (set, '{', '}', False) + frozenset_info = (frozenset, 'frozenset({', '})', False) + else: + string_types = (str, unicode) + set_info = (set, 'set([', '])', False) + frozenset_info = (frozenset, 'frozenset([', '])', False) + + # Collection types are recursively iterated for each limit in + # maxcollection. + maxcollection = (15, 10) + + # Specifies type, prefix string, suffix string, and whether to include a + # comma if there is only one element. (Using a sequence rather than a + # mapping because we use isinstance() to determine the matching type.) + collection_types = [ + (tuple, '(', ')', True), + (list, '[', ']', False), + frozenset_info, + set_info, + ] + try: + from collections import deque + collection_types.append((deque, 'deque([', '])', False)) + except: + pass + + # type, prefix string, suffix string, item prefix string, item key/value separator, item suffix string + dict_types = [(dict, '{', '}', '', ': ', '')] + try: + from collections import OrderedDict + dict_types.append((OrderedDict, 'OrderedDict([', '])', '(', ', ', ')')) + except: + pass + + # All other types are treated identically to strings, but using + # different limits. + maxother_outer = 2 ** 16 + maxother_inner = 30 + + def __call__(self, obj): + try: + return ''.join(self._repr(obj, 0)) + except: + try: + return 'An exception was raised: %r' % sys.exc_info()[1] + except: + return 'An exception was raised' + + def _repr(self, obj, level): + '''Returns an iterable of the parts in the final repr string.''' + + try: + obj_repr = type(obj).__repr__ + except: + obj_repr = None + + def has_obj_repr(t): + r = t.__repr__ + try: + return obj_repr == r + except: + return obj_repr is r + + for t, prefix, suffix, comma in self.collection_types: + if isinstance(obj, t) and has_obj_repr(t): + return self._repr_iter(obj, level, prefix, suffix, comma) + + for t, prefix, suffix, item_prefix, item_sep, item_suffix in self.dict_types: + if isinstance(obj, t) and has_obj_repr(t): + return self._repr_dict(obj, level, prefix, suffix, item_prefix, item_sep, item_suffix) + + for t in self.string_types: + if isinstance(obj, t) and has_obj_repr(t): + return self._repr_str(obj, level) + + if self._is_long_iter(obj): + return self._repr_long_iter(obj) + + return self._repr_other(obj, level) + + # Determines whether an iterable exceeds the limits set in maxlimits, and is therefore unsafe to repr(). + def _is_long_iter(self, obj, level = 0): + try: + # Strings have their own limits (and do not nest). Because they don't have __iter__ in 2.x, this + # check goes before the next one. + if isinstance(obj, self.string_types): + return len(obj) > self.maxstring_inner + + # If it's not an iterable (and not a string), it's fine. + if not hasattr(obj, '__iter__'): + return False + + # Iterable is its own iterator - this is a one-off iterable like generator or enumerate(). We can't + # really count that, but repr() for these should not include any elements anyway, so we can treat it + # the same as non-iterables. + if obj is iter(obj): + return False + + # xrange reprs fine regardless of length. + if isinstance(obj, xrange): + return False + + # numpy and scipy collections (ndarray etc) have self-truncating repr, so they're always safe. + try: + module = type(obj).__module__.partition('.')[0] + if module in ('numpy', 'scipy'): + return False + except: + pass + + # Iterables that nest too deep are considered long. + if level >= len(self.maxcollection): + return True + + # It is too long if the length exceeds the limit, or any of its elements are long iterables. + if hasattr(obj, '__len__'): + try: + l = len(obj) + except: + l = None + if l is not None and l > self.maxcollection[level]: + return True + return any((self._is_long_iter(item, level + 1) for item in obj)) + return any(i > self.maxcollection[level] or self._is_long_iter(item, level + 1) for i, item in enumerate(obj)) + + except: + # If anything breaks, assume the worst case. + return True + + def _repr_iter(self, obj, level, prefix, suffix, comma_after_single_element = False): + yield prefix + + if level >= len(self.maxcollection): + yield '...' + else: + count = self.maxcollection[level] + yield_comma = False + for item in obj: + if yield_comma: + yield ', ' + yield_comma = True + + count -= 1 + if count <= 0: + yield '...' + break + + for p in self._repr(item, 100 if item is obj else level + 1): + yield p + else: + if comma_after_single_element and count == self.maxcollection[level] - 1: + yield ',' + yield suffix + + def _repr_long_iter(self, obj): + try: + obj_repr = '<%s, len() = %s>' % (type(obj).__name__, len(obj)) + except: + try: + obj_repr = '<' + type(obj).__name__ + '>' + except: + obj_repr = '' + yield obj_repr + + def _repr_dict(self, obj, level, prefix, suffix, item_prefix, item_sep, item_suffix): + if not obj: + yield prefix + suffix + return + if level >= len(self.maxcollection): + yield prefix + '...' + suffix + return + + yield prefix + + count = self.maxcollection[level] + yield_comma = False + + try: + sorted_keys = sorted(obj) + except Exception: + sorted_keys = list(obj) + + for key in sorted_keys: + if yield_comma: + yield ', ' + yield_comma = True + + count -= 1 + if count <= 0: + yield '...' + break + + yield item_prefix + for p in self._repr(key, level + 1): + yield p + + yield item_sep + + try: + item = obj[key] + except Exception: + yield '' + else: + for p in self._repr(item, 100 if item is obj else level + 1): + yield p + yield item_suffix + + yield suffix + + def _repr_str(self, obj, level): + return self._repr_obj(obj, level, self.maxstring_inner, self.maxstring_outer) + + def _repr_other(self, obj, level): + return self._repr_obj(obj, level, self.maxother_inner, self.maxother_outer) + + def _repr_obj(self, obj, level, limit_inner, limit_outer): + try: + obj_repr = repr(obj) + except: + try: + obj_repr = object.__repr__(obj) + except: + try: + obj_repr = '' + except: + obj_repr = '' + + limit = limit_inner if level > 0 else limit_outer + + if limit >= len(obj_repr): + yield obj_repr + return + + # Slightly imprecise calculations - we may end up with a string that is + # up to 3 characters longer than limit. If you need precise formatting, + # you are using the wrong class. + left_count, right_count = max(1, int(2 * limit / 3)), max(1, int(limit / 3)) + + yield obj_repr[:left_count] + yield '...' + yield obj_repr[-right_count:] + + + def _selftest(self): + # Test the string limiting somewhat automatically + tests = [] + tests.append((7, 9, 'A' * (5))) + tests.append((self.maxstring_outer + 3, self.maxstring_inner + 3 + 2, 'A' * (self.maxstring_outer + 10))) + if sys.version_info >= (3, 0): + tests.append((self.maxstring_outer + 4, self.maxstring_inner + 4 + 2, bytes('A', 'ascii') * (self.maxstring_outer + 10))) + else: + tests.append((self.maxstring_outer + 4, self.maxstring_inner + 4 + 2, unicode('A') * (self.maxstring_outer + 10))) + + for limit1, limit2, value in tests: + assert len(self(value)) <= limit1 <= len(repr(value)), (len(self(value)), limit1, len(repr(value)), value) + assert len(self([value])) <= limit2 <= len(repr([value])), (len(self([value])), limit2, len(repr([value])), self([value])) + + def test(source, expected): + actual = self(source) + if actual != expected: + print("Source " + repr(source)) + print("Expect " + expected) + print("Actual " + actual) + print("") + assert False + + def re_test(source, pattern): + import re + actual = self(source) + if not re.match(pattern, actual): + print("Source " + repr(source)) + print("Pattern " + pattern) + print("Actual " + actual) + print("") + assert False + + for ctype, _prefix, _suffix, comma in self.collection_types: + for i in range(len(self.maxcollection)): + prefix = _prefix * (i + 1) + if comma: + suffix = _suffix + ("," + _suffix) * i + else: + suffix = _suffix * (i + 1) + #print("ctype = " + ctype.__name__ + ", maxcollection[" + str(i) + "] == " + str(self.maxcollection[i])) + c1 = ctype(range(self.maxcollection[i] - 1)) + inner_repr = prefix + ', '.join(str(j) for j in c1) + c2 = ctype(range(self.maxcollection[i])) + c3 = ctype(range(self.maxcollection[i] + 1)) + for j in range(i): + c1, c2, c3 = ctype((c1,)), ctype((c2,)), ctype((c3,)) + test(c1, inner_repr + suffix) + test(c2, inner_repr + ", ..." + suffix) + test(c3, inner_repr + ", ..." + suffix) + + if ctype is set: + # Cannot recursively add sets to sets + break + + # Assume that all tests apply equally to all iterable types and only + # test with lists. + c1 = list(range(self.maxcollection[0] * 2)) + c2 = [c1 for _ in range(self.maxcollection[0] * 2)] + c1_expect = '[' + ', '.join(str(j) for j in range(self.maxcollection[0] - 1)) + ', ...]' + test(c1, c1_expect) + c1_expect2 = '[' + ', '.join(str(j) for j in range(self.maxcollection[1] - 1)) + ', ...]' + c2_expect = '[' + ', '.join(c1_expect2 for _ in range(self.maxcollection[0] - 1)) + ', ...]' + test(c2, c2_expect) + + # Ensure dict keys and values are limited correctly + d1 = {} + d1_key = 'a' * self.maxstring_inner * 2 + d1[d1_key] = d1_key + re_test(d1, "{'a+\.\.\.a+': 'a+\.\.\.a+'}") + d2 = {d1_key : d1} + re_test(d2, "{'a+\.\.\.a+': {'a+\.\.\.a+': 'a+\.\.\.a+'}}") + d3 = {d1_key : d2} + if len(self.maxcollection) == 2: + re_test(d3, "{'a+\.\.\.a+': {'a+\.\.\.a+': {\.\.\.}}}") + else: + re_test(d3, "{'a+\.\.\.a+': {'a+\.\.\.a+': {'a+\.\.\.a+': 'a+\.\.\.a+'}}}") + + # Ensure empty dicts work + test({}, '{}') + + # Ensure dict keys are sorted + d1 = {} + d1['c'] = None + d1['b'] = None + d1['a'] = None + test(d1, "{'a': None, 'b': None, 'c': None}") + + if sys.version_info >= (3, 0): + # Ensure dicts with unsortable keys do not crash + d1 = {} + for _ in range(100): + d1[object()] = None + try: + list(sorted(d1)) + assert False, "d1.keys() should be unorderable" + except TypeError: + pass + self(d1) + + # Test with objects with broken repr implementations + class TestClass(object): + def __repr__(self): + raise NameError + try: + repr(TestClass()) + assert False, "TestClass().__repr__ should have thrown" + except NameError: + pass + self(TestClass()) + + # Test with objects with long repr implementations + class TestClass(object): + repr_str = '<' + 'A' * self.maxother_outer * 2 + '>' + def __repr__(self): + return self.repr_str + re_test(TestClass(), r'\') + + # Test collections that don't override repr + class TestClass(dict): pass + test(TestClass(), '{}') + class TestClass(list): pass + test(TestClass(), '[]') + + # Test collections that override repr + class TestClass(dict): + def __repr__(self): return 'MyRepr' + test(TestClass(), 'MyRepr') + class TestClass(list): + def __init__(self, iter = ()): list.__init__(self, iter) + def __repr__(self): return 'MyRepr' + test(TestClass(), 'MyRepr') + + # Test collections and iterables with long repr + test(TestClass(xrange(0, 15)), 'MyRepr') + test(TestClass(xrange(0, 16)), '') + test(TestClass([TestClass(xrange(0, 10))]), 'MyRepr') + test(TestClass([TestClass(xrange(0, 11))]), '') + + # Test strings inside long iterables + test(TestClass(['a' * (self.maxcollection[1] + 1)]), 'MyRepr') + test(TestClass(['a' * (self.maxstring_inner + 1)]), '') + + # Test range + if sys.version[0] == '2': + range_name = 'xrange' + else: + range_name = 'range' + test(xrange(1, self.maxcollection[0] + 1), '%s(1, %s)' % (range_name, self.maxcollection[0] + 1)) + + # Test directly recursive collections + c1 = [1, 2] + c1.append(c1) + test(c1, '[1, 2, [...]]') + d1 = {1: None} + d1[2] = d1 + test(d1, '{1: None, 2: {...}}') + + # Find the largest possible repr and ensure it is below our arbitrary + # limit (8KB). + coll = '-' * (self.maxstring_outer * 2) + for limit in reversed(self.maxcollection[1:]): + coll = [coll] * (limit * 2) + dcoll = {} + for i in range(self.maxcollection[0]): + dcoll[str(i) * self.maxstring_outer] = coll + text = self(dcoll) + #try: + # text_repr = repr(dcoll) + #except MemoryError: + # print('Memory error raised while creating repr of test data') + # text_repr = '' + #print('len(SafeRepr()(dcoll)) = ' + str(len(text)) + ', len(repr(coll)) = ' + str(len(text_repr))) + assert len(text) < 8192 + + # Test numpy types - they should all use their native reprs, even arrays exceeding limits + try: + import numpy as np + except ImportError: + print('WARNING! could not import numpy - skipping all numpy tests.') + else: + test(np.int32(123), repr(np.int32(123))) + test(np.float64(123.456), repr(np.float64(123.456))) + test(np.zeros(self.maxcollection[0] + 1), repr(np.zeros(self.maxcollection[0] + 1))); + +if __name__ == '__main__': + print('Running tests...') + SafeRepr()._selftest() diff --git a/src/client/debugger/commandRunner.ts b/src/client/debugger/pdb/commandRunner.ts similarity index 94% rename from src/client/debugger/commandRunner.ts rename to src/client/debugger/pdb/commandRunner.ts index 42725eee511d..f86ac4f48d3f 100644 --- a/src/client/debugger/commandRunner.ts +++ b/src/client/debugger/pdb/commandRunner.ts @@ -155,14 +155,6 @@ export class CommandRunner { return; } - // var lastCmd = this.executingCommands[this.executingCommands.length - 1]; - // var isPrompt = lastLine === lastCmd.prompt; - - // if (isPrompt) { - // this.outputBuffer = ""; - // this.pythonProc.stdin.write(lastCmd.promptResponse + "\n"); - // return; - // } if (isEndOfLine) { var lastCmd = this.executingCommands[this.executingCommands.length - 1]; //Remove (Pdb) prompt diff --git a/src/client/debugger/common.ts b/src/client/debugger/pdb/common.ts similarity index 100% rename from src/client/debugger/common.ts rename to src/client/debugger/pdb/common.ts diff --git a/src/client/debugger/debuggerMain.ts b/src/client/debugger/pdb/debuggerMain.ts similarity index 100% rename from src/client/debugger/debuggerMain.ts rename to src/client/debugger/pdb/debuggerMain.ts diff --git a/src/client/debugger/main.ts b/src/client/debugger/pdb/main.ts similarity index 99% rename from src/client/debugger/main.ts rename to src/client/debugger/pdb/main.ts index e5c6c9fca4f2..7c147c25522e 100644 --- a/src/client/debugger/main.ts +++ b/src/client/debugger/pdb/main.ts @@ -129,7 +129,6 @@ class PythonDebugSession extends DebugSession { private launchResponse: DebugProtocol.LaunchResponse; protected launchRequest(response: DebugProtocol.LaunchResponse, args: LaunchRequestArguments): void { this._sourceFile = args.program; - //this._sourceLines = readFileSync(this._sourceFile).toString().split('\n'); if (args.stopOnEntry) { this.launchResponse = response; @@ -146,7 +145,7 @@ class PythonDebugSession extends DebugSession { } var programArgs = Array.isArray(args.args) ? args.args : []; - this.pdbRunner = new PdbRunner(this._sourceFile, this, null, programArgs, pythonPath, args.stopOnEntry); + this.pdbRunner = new PdbRunner(this._sourceFile, this, programArgs, pythonPath, args.stopOnEntry); this.pdbRunner.pdbLoaded.then(() => { this.sendResponse(this.launchResponse); diff --git a/src/client/debugger/pdb.ts b/src/client/debugger/pdb/pdb.ts similarity index 96% rename from src/client/debugger/pdb.ts rename to src/client/debugger/pdb/pdb.ts index 169af39893e1..b453f393ae86 100644 --- a/src/client/debugger/pdb.ts +++ b/src/client/debugger/pdb/pdb.ts @@ -11,7 +11,6 @@ import * as fs from 'fs'; import * as child_process from 'child_process'; import * as StringDecoder from 'string_decoder'; import {DebugSession, InitializedEvent, TerminatedEvent, StoppedEvent, OutputEvent, Thread, StackFrame, Scope, Source, Handles} from 'vscode-debugadapter'; -import * as settings from './../common/configSettings'; function ignoreEmpty(line) { return line.trim() !== "" && line.trim() !== "(Pdb)" && line.trim() !== "\n" && line.trim() !== "\r" && line.trim() !== "\r\n"; @@ -40,14 +39,12 @@ export class PdbRunner { private pythonProc: child_process.ChildProcess;//pty.Terminal; private _sourceFile: string; private debugSession: DebugSession; - private settings: settings.IPythonSettings; private debugArgs: string[]; private pythonPath: string; private stopOnEntry: boolean; - public constructor(sourceFile: string, debugSession: DebugSession, settings: settings.IPythonSettings, args: string[], pythonPath: string, stopOnEntry?: boolean) { + public constructor(sourceFile: string, debugSession: DebugSession, args: string[], pythonPath: string, stopOnEntry?: boolean) { this._sourceFile = sourceFile; this.debugSession = debugSession; - this.settings = settings; this.debugArgs = args; this.pythonPath = pythonPath; this.stopOnEntry = stopOnEntry === true; diff --git a/src/client/debugger/pdbRunner.ts b/src/client/debugger/pdb/pdbRunner.ts similarity index 100% rename from src/client/debugger/pdbRunner.ts rename to src/client/debugger/pdb/pdbRunner.ts diff --git a/src/client/debugger/vs/Common/BufferReader.ts b/src/client/debugger/vs/Common/BufferReader.ts new file mode 100644 index 000000000000..be7e4e4ae1cb --- /dev/null +++ b/src/client/debugger/vs/Common/BufferReader.ts @@ -0,0 +1,170 @@ +export class BufferReader { + private buffer: Buffer; + private isInTransaction: boolean; + private bytesRead: number = 0; + + constructor(buffer: Buffer) { + this.buffer = buffer; + } + + public BeginTransaction() { + this.isInTransaction = true; + this.bytesRead = 0; + this.ClearErrors(); + } + public EndTransaction() { + this.isInTransaction = true; + this.buffer = this.buffer.slice(this.bytesRead); + this.bytesRead = 0; + this.ClearErrors(); + } + public RollBackTransaction() { + this.isInTransaction = false; + this.bytesRead = 0; + this.ClearErrors(); + } + + public ClearErrors() { + this.hasInsufficientDataForReading = false; + } + + private hasInsufficientDataForReading: boolean = false; + public get HasInsufficientDataForReading(): boolean { + return this.hasInsufficientDataForReading; + } + + public toString(): string { + return this.buffer.toString(); + } + + public get Length(): number { + return this.buffer.length; + } + + public Append(additionalData: Buffer) { + if (this.buffer.length === 0) { + this.buffer = additionalData; + return; + } + var newBuffer = new Buffer(this.buffer.length + additionalData.length); + this.buffer.copy(newBuffer); + additionalData.copy(newBuffer, this.buffer.length) + this.buffer = newBuffer; + } + + private isSufficientDataAvailable(length: number): boolean { + if (this.buffer.length < (this.bytesRead + length)) { + this.hasInsufficientDataForReading = true; + } + + return !this.hasInsufficientDataForReading; + } + + public ReadByte(): number { + if (!this.isSufficientDataAvailable(1)) { + return null; + } + + var value = this.buffer.slice(this.bytesRead, this.bytesRead + 1)[0]; + if (this.isInTransaction) { + this.bytesRead++; + } + else { + this.buffer = this.buffer.slice(1); + } + return value; + } + + public ReadString(): string { + var byteRead = this.ReadByte(); + if (this.HasInsufficientDataForReading) { + return null; + } + + if (byteRead < 0) { + console.error("Socket.ReadString failed to read string type"); + throw new Error("IOException();"); + } + + var type = new Buffer([byteRead]).toString(); + var isUnicode = false; + switch (type) { + case 'N': // null string + return null; + case 'U': + isUnicode = true; + break; + case 'A': { + isUnicode = false; + break; + } + default: { + console.error("Socket.ReadString failed to parse unknown string type " + type); + throw new Error("IOException();"); + } + } + + var len = this.ReadInt32(); + if (this.HasInsufficientDataForReading) { + return null; + } + + if (!this.isSufficientDataAvailable(len)) { + return null; + } + + var stringBuffer = this.buffer.slice(this.bytesRead, this.bytesRead + len); + if (this.isInTransaction) { + this.bytesRead = this.bytesRead + len; + } + else { + this.buffer = this.buffer.slice(len); + } + + var resp = isUnicode ? stringBuffer.toString('utf-8') : stringBuffer.toString(); + return resp; + } + + public ReadInt32(): number { + return this.ReadInt64(); + } + + public ReadInt64(): number { + if (!this.isSufficientDataAvailable(8)) { + return null; + } + + var buf = this.buffer.slice(this.bytesRead, this.bytesRead + 8); + + if (this.isInTransaction) { + this.bytesRead = this.bytesRead + 8; + } + else { + this.buffer = this.buffer.slice(8); + } + + // Can't use BitConverter because we need to convert big-endian to little-endian here, + // and BitConverter.IsLittleEndian is platform-dependent (and usually true). + var hi = ((buf[0] << 0x18) | (buf[1] << 0x10) | (buf[2] << 0x08) | (buf[3] << 0x00)); + var lo = ((buf[4] << 0x18) | (buf[5] << 0x10) | (buf[6] << 0x08) | (buf[7] << 0x00)); + var returnValue = ((hi << 0x20) | lo); + + return returnValue; + } + + public ReadAsciiString(length: number): string { + if (!this.isSufficientDataAvailable(length)) { + return null; + } + + var stringBuffer = this.buffer.slice(this.bytesRead, this.bytesRead + length); + if (this.isInTransaction) { + this.bytesRead = this.bytesRead + length; + } + else { + this.buffer = this.buffer.slice(length); + } + return stringBuffer.toString("ascii"); + } +} + diff --git a/src/client/debugger/vs/Common/BufferUtils.ts b/src/client/debugger/vs/Common/BufferUtils.ts new file mode 100644 index 000000000000..996809bb13cf --- /dev/null +++ b/src/client/debugger/vs/Common/BufferUtils.ts @@ -0,0 +1,28 @@ +// var long = require("long"); +// +// export class BufferUtils { +// public static fromInt64(num: number): Buffer { +// //convert to long +// var longNumber = long.fromNumber(num, true); +// +// // Can't use BitConverter because we need to convert big-endian to little-endian here, +// // and BitConverter.IsLittleEndian is platform-dependent (and usually true). +// var hi = longNumber.shiftRight(0x20).toInt() +// var lo = longNumber.and(0xFFFFFFFF).toInt(); +// var buf = [ +// ((hi >> 0x18) & 0xFF), +// ((hi >> 0x10) & 0xFF), +// ((hi >> 0x08) & 0xFF), +// ((hi >> 0x00) & 0xFF), +// ((lo >> 0x18) & 0xFF), +// ((lo >> 0x10) & 0xFF), +// ((lo >> 0x08) & 0xFF), +// ((lo >> 0x00) & 0xFF) +// ]; +// +// return new Buffer(buf); +// } +// public static fromInt32(num: number): Buffer { +// return this.fromInt64(num); +// } +// } \ No newline at end of file diff --git a/src/client/debugger/vs/Common/Contracts.ts b/src/client/debugger/vs/Common/Contracts.ts new file mode 100644 index 000000000000..b1d7a5be0dd2 --- /dev/null +++ b/src/client/debugger/vs/Common/Contracts.ts @@ -0,0 +1,157 @@ +export interface IDebugServer { + port: number; +} + +export enum FrameKind { + None, + Python, + Django +}; + +export enum PythonLanguageVersion { + Is2, + Is3 +} + +export enum PythonEvaluationResultReprKind { + Normal, + Raw, + RawLen +} + +export enum PythonEvaluationResultFlags { + None = 0, + Expandable = 1, + MethodCall = 2, + SideEffects = 4, + Raw = 8, + HasRawRepr = 16, +} + +export interface IPythonProcess extends NodeJS.EventEmitter { + SendStepInto(threadId: number); + SendStepOver(threadId: number): Promise; + SendStepOut(threadId: number); + SendResumeThread(threadId: number); + AutoResumeThread(threadId: number); + SendClearStepping(threadId: number); + ExecuteText(text: string, reprKind: any, stackFrame: IPythonStackFrame): Promise; + EnumChildren(text: string, stackFrame: IPythonStackFrame, timeout: number): Promise; + SetLineNumber(pythonStackFrame: IPythonStackFrame, lineNo: number); + Threads: Map; + + //TODO:Fix this, shouldn't be exposed + PendingChildEnumCommands: Map; + PendingExecuteCommands: Map; +} + +export interface IPythonEvaluationResult { + Flags: PythonEvaluationResultFlags; + IsExpandable: boolean; + StringRepr: string; + HexRepr: string; + TypeName: string; + Length: number; + ExceptionText?: string; + Expression: string; + ChildName: string; + Process: IPythonProcess; + Frame: IPythonStackFrame; +} + + +export interface IPythonModule { + ModuleId: number; + Name: string; + Filename: string; +} + + +export interface IPythonThread { + IsWorkerThread: boolean; + Process: IPythonProcess; + Name: string; + Id: number; + Frames: IPythonStackFrame[]; +} + +export interface IPythonStackFrame { + StartLine: number; + EndLine: number; + Thread: IPythonThread; + LineNo: number; + FunctionName: string; + FileName: string; + Kind: FrameKind; + FrameId: number; + Locals: IPythonEvaluationResult[]; + Parameters: IPythonEvaluationResult[]; +} + +export interface IDjangoStackFrame extends IPythonStackFrame { + SourceFile: string; + SourceLine: number; +} + +export interface IStepCommand { + PromiseResolve: (pyThread: IPythonThread) => void; + PythonThreadId: number; +} + +export interface IBreakpointCommand { + Id: number; + PromiseResolve: () => void; + PromiseReject: () => void; +} +export interface IChildEnumCommand { + Id: number; + Frame: IPythonStackFrame; + PromiseResolve: (value: IPythonEvaluationResult[]) => void; + PromiseReject: () => void; +} +export interface IExecutionCommand { + Id: number; + Text: string; + Frame: IPythonStackFrame; + PromiseResolve: (value: IPythonEvaluationResult) => void; + PromiseReject: (error: string) => void; +} +// Must be in sync with BREAKPOINT_CONDITION_* constants in visualstudio_py_debugger.py. +export enum PythonBreakpointConditionKind { + Always = 0, + WhenTrue = 1, + WhenChanged = 2 +} + +// Must be in sync with BREAKPOINT_PASS_COUNT_* constants in visualstudio_py_debugger.py. +export enum PythonBreakpointPassCountKind { + Always = 0, + Every = 1, + WhenEqual = 2, + WhenEqualOrGreater = 3 +} + +export interface IPythonBreakpoint { + IsDjangoBreakpoint?: boolean; + Id: number; + Filename: string; + LineNo: number; + ConditionKind: PythonBreakpointConditionKind; + Condition: string; + PassCountKind: PythonBreakpointPassCountKind; + PassCount: number; +} +export interface IPythonException { + TypeName: string; + Description: string; +} + +export enum StreamDataType { + Int32, + Int64, + String +} +export interface IStreamData { + DataType: StreamDataType; + RawData: any; +} \ No newline at end of file diff --git a/src/client/debugger/vs/Common/SocketStream.ts b/src/client/debugger/vs/Common/SocketStream.ts new file mode 100644 index 000000000000..8b2f8c61e49b --- /dev/null +++ b/src/client/debugger/vs/Common/SocketStream.ts @@ -0,0 +1,214 @@ +import * as net from 'net'; +var long = require("long"); + + +export class SocketStream { + constructor(socket: net.Socket, buffer: Buffer) { + this.buffer = buffer; + this.socket = socket; + } + + private socket: net.Socket; + public WriteInt32(num: number) { + this.WriteInt64(num); + } + + public WriteInt64(num: number) { + //convert to long + var longNumber = long.fromNumber(num, true); + + // Can't use BitConverter because we need to convert big-endian to little-endian here, + // and BitConverter.IsLittleEndian is platform-dependent (and usually true). + var hi = longNumber.shiftRight(0x20).toInt() + var lo = longNumber.and(0xFFFFFFFF).toInt(); + var buf = [ + ((hi >> 0x18) & 0xFF), + ((hi >> 0x10) & 0xFF), + ((hi >> 0x08) & 0xFF), + ((hi >> 0x00) & 0xFF), + ((lo >> 0x18) & 0xFF), + ((lo >> 0x10) & 0xFF), + ((lo >> 0x08) & 0xFF), + ((lo >> 0x00) & 0xFF) + ]; + + var buffer = new Buffer(buf); + this.socket.write(buffer); + } + public WriteString(value: string) { + var stringBuffer = new Buffer(value, "utf-8"); + this.WriteInt32(stringBuffer.length); + if (stringBuffer.length > 0) { + this.socket.write(stringBuffer); + } + } + public Write(buffer: Buffer) { + this.socket.write(buffer); + } + + + private buffer: Buffer; + private isInTransaction: boolean; + private bytesRead: number = 0; + + public BeginTransaction() { + this.isInTransaction = true; + this.bytesRead = 0; + this.ClearErrors(); + } + public EndTransaction() { + this.isInTransaction = true; + this.buffer = this.buffer.slice(this.bytesRead); + this.bytesRead = 0; + this.ClearErrors(); + } + public RollBackTransaction() { + this.isInTransaction = false; + this.bytesRead = 0; + this.ClearErrors(); + } + + public ClearErrors() { + this.hasInsufficientDataForReading = false; + } + + private hasInsufficientDataForReading: boolean = false; + public get HasInsufficientDataForReading(): boolean { + return this.hasInsufficientDataForReading; + } + + public toString(): string { + return this.buffer.toString(); + } + + public get Length(): number { + return this.buffer.length; + } + + public Append(additionalData: Buffer) { + if (this.buffer.length === 0) { + this.buffer = additionalData; + return; + } + var newBuffer = new Buffer(this.buffer.length + additionalData.length); + this.buffer.copy(newBuffer); + additionalData.copy(newBuffer, this.buffer.length) + this.buffer = newBuffer; + } + + private isSufficientDataAvailable(length: number): boolean { + if (this.buffer.length < (this.bytesRead + length)) { + this.hasInsufficientDataForReading = true; + } + + return !this.hasInsufficientDataForReading; + } + + public ReadByte(): number { + if (!this.isSufficientDataAvailable(1)) { + return null; + } + + var value = this.buffer.slice(this.bytesRead, this.bytesRead + 1)[0]; + if (this.isInTransaction) { + this.bytesRead++; + } + else { + this.buffer = this.buffer.slice(1); + } + return value; + } + + public ReadString(): string { + var byteRead = this.ReadByte(); + if (this.HasInsufficientDataForReading) { + return null; + } + + if (byteRead < 0) { + console.error("Socket.ReadString failed to read string type"); + throw new Error("IOException();"); + } + + var type = new Buffer([byteRead]).toString(); + var isUnicode = false; + switch (type) { + case 'N': // null string + return null; + case 'U': + isUnicode = true; + break; + case 'A': { + isUnicode = false; + break; + } + default: { + console.error("Socket.ReadString failed to parse unknown string type " + type); + throw new Error("IOException();"); + } + } + + var len = this.ReadInt32(); + if (this.HasInsufficientDataForReading) { + return null; + } + + if (!this.isSufficientDataAvailable(len)) { + return null; + } + + var stringBuffer = this.buffer.slice(this.bytesRead, this.bytesRead + len); + if (this.isInTransaction) { + this.bytesRead = this.bytesRead + len; + } + else { + this.buffer = this.buffer.slice(len); + } + + var resp = isUnicode ? stringBuffer.toString('utf-8') : stringBuffer.toString(); + return resp; + } + + public ReadInt32(): number { + return this.ReadInt64(); + } + + public ReadInt64(): number { + if (!this.isSufficientDataAvailable(8)) { + return null; + } + + var buf = this.buffer.slice(this.bytesRead, this.bytesRead + 8); + + if (this.isInTransaction) { + this.bytesRead = this.bytesRead + 8; + } + else { + this.buffer = this.buffer.slice(8); + } + + // Can't use BitConverter because we need to convert big-endian to little-endian here, + // and BitConverter.IsLittleEndian is platform-dependent (and usually true). + var hi = ((buf[0] << 0x18) | (buf[1] << 0x10) | (buf[2] << 0x08) | (buf[3] << 0x00)); + var lo = ((buf[4] << 0x18) | (buf[5] << 0x10) | (buf[6] << 0x08) | (buf[7] << 0x00)); + var returnValue = ((hi << 0x20) | lo); + + return returnValue; + } + + public ReadAsciiString(length: number): string { + if (!this.isSufficientDataAvailable(length)) { + return null; + } + + var stringBuffer = this.buffer.slice(this.bytesRead, this.bytesRead + length); + if (this.isInTransaction) { + this.bytesRead = this.bytesRead + length; + } + else { + this.buffer = this.buffer.slice(length); + } + return stringBuffer.toString("ascii"); + } +} + diff --git a/src/client/debugger/vs/Common/Utils.ts b/src/client/debugger/vs/Common/Utils.ts new file mode 100644 index 000000000000..d75e00342970 --- /dev/null +++ b/src/client/debugger/vs/Common/Utils.ts @@ -0,0 +1,60 @@ +import {IPythonProcess, IPythonThread, IPythonModule, IPythonEvaluationResult} from './Contracts'; +import * as path from 'path'; + +export function CreatePythonThread(id: number, isWorker: boolean, process: IPythonProcess, name: string = ""): IPythonThread { + return { + IsWorkerThread: isWorker, + Process: process, + Name: name, + Id: id, + Frames: [] + }; +} + +export function CreatePythonModule(id: number, fileName: string): IPythonModule { + var name = fileName; + if (typeof fileName == "string") { + try { + name = path.basename(fileName); + } + catch (ex) { + } + } + else { + name = ""; + } + + return { + ModuleId: id, + Name: name, + Filename: fileName + }; +} + +export function FixupEscapedUnicodeChars(value: string): string { + return value; +} + +export class IdDispenser { + private _freedInts: number[] = []; + private _curValue: number = 0; + + public Allocate(): number { + if (this._freedInts.length > 0) { + var res: number = this._freedInts[this._freedInts.length - 1]; + this._freedInts.splice(this._freedInts.length - 1, 1); + return res; + } else { + var res: number = this._curValue++; + return res; + } + } + + public Free(id: number) { + if (id + 1 == this._curValue) { + this._curValue--; + } else { + this._freedInts.push(id); + } + } +} \ No newline at end of file diff --git a/src/client/debugger/vs/ProxyCommands.ts b/src/client/debugger/vs/ProxyCommands.ts new file mode 100644 index 000000000000..537b8b2cca64 --- /dev/null +++ b/src/client/debugger/vs/ProxyCommands.ts @@ -0,0 +1,29 @@ +export class Commands { + public static ExitCommandBytes: Buffer = new Buffer("exit"); + public static StepIntoCommandBytes: Buffer = new Buffer("stpi"); + public static StepOutCommandBytes: Buffer = new Buffer("stpo"); + public static StepOverCommandBytes: Buffer = new Buffer("stpv"); + public static BreakAllCommandBytes: Buffer = new Buffer("brka"); + public static SetBreakPointCommandBytes: Buffer = new Buffer("brkp"); + public static SetBreakPointConditionCommandBytes: Buffer = new Buffer("brkc"); + public static SetBreakPointPassCountCommandBytes: Buffer = new Buffer("bkpc"); + public static GetBreakPointHitCountCommandBytes: Buffer = new Buffer("bkgh"); + public static SetBreakPointHitCountCommandBytes: Buffer = new Buffer("bksh"); + public static RemoveBreakPointCommandBytes: Buffer = new Buffer("brkr"); + public static ResumeAllCommandBytes: Buffer = new Buffer("resa"); + public static GetThreadFramesCommandBytes: Buffer = new Buffer("thrf"); + public static ExecuteTextCommandBytes: Buffer = new Buffer("exec"); + public static ResumeThreadCommandBytes: Buffer = new Buffer("rest"); + public static AutoResumeThreadCommandBytes: Buffer = new Buffer("ares"); + public static ClearSteppingCommandBytes: Buffer = new Buffer("clst"); + public static SetLineNumberCommand: Buffer = new Buffer("setl"); + public static GetChildrenCommandBytes: Buffer = new Buffer("chld"); + public static DetachCommandBytes: Buffer = new Buffer("detc"); + public static SetExceptionInfoCommandBytes: Buffer = new Buffer("sexi"); + public static SetExceptionHandlerInfoCommandBytes: Buffer = new Buffer("sehi"); + public static RemoveDjangoBreakPointCommandBytes: Buffer = new Buffer("bkdr"); + public static AddDjangoBreakPointCommandBytes: Buffer = new Buffer("bkda"); + public static ConnectReplCommandBytes: Buffer = new Buffer("crep"); + public static DisconnectReplCommandBytes: Buffer = new Buffer("drep"); + public static LastAckCommandBytes: Buffer = new Buffer("lack"); +} diff --git a/src/client/debugger/vs/PythonProcess.ts b/src/client/debugger/vs/PythonProcess.ts new file mode 100644 index 000000000000..356a1a460392 --- /dev/null +++ b/src/client/debugger/vs/PythonProcess.ts @@ -0,0 +1,289 @@ +"use string"; + +import * as net from 'net'; +import {EventEmitter} from 'events'; +import {FrameKind, IPythonProcess, IPythonThread, IPythonModule, IPythonEvaluationResult, IPythonStackFrame, IStepCommand} from './Common/Contracts'; +import {IPythonBreakpoint, PythonBreakpointConditionKind, PythonBreakpointPassCountKind, IBreakpointCommand, IChildEnumCommand} from './Common/Contracts'; +import {PythonEvaluationResultReprKind, IExecutionCommand} from './Common/Contracts'; +import {Commands} from './ProxyCommands'; +import * as utils from './Common/Utils'; +import {PythonProcessCallbackHandler} from './PythonProcessCallbackHandler'; +import {SocketStream} from './Common/SocketStream'; + +export class PythonProcess extends EventEmitter implements IPythonProcess { + private id: number; + public get Id(): number { + return this.id; + } + private guid: string; + public get Guid(): string { + return this.guid; + } + + private hasExited: boolean; + public get HasExited(): boolean { + return this.hasExited; + } + + private _mainThread: IPythonThread; + public get MainThread(): IPythonThread { + return this._mainThread; + } + + private _lastExecutedThread: IPythonThread; + public get LastExecutedThread(): IPythonThread { + return this._lastExecutedThread; + } + private _idDispenser: utils.IdDispenser; + private _threads: Map; + public get Threads(): Map { + return this._threads; + } + + public PendingChildEnumCommands: Map; + public PendingExecuteCommands: Map; + private callbackHandler: PythonProcessCallbackHandler; + private stream: SocketStream; + constructor(id: number, guid: string) { + super(); + this.id = id; + this.guid = guid; + this._threads = new Map(); + this._idDispenser = new utils.IdDispenser(); + this.PendingChildEnumCommands = new Map(); + this.PendingExecuteCommands = new Map(); + } + + public Terminate() { + } + + public Connect(buffer: Buffer, socket: net.Socket) { + this.stream = new SocketStream(socket, buffer); + + var guid = this.stream.ReadString(); + var result = this.stream.ReadInt32(); + console.log(guid); + console.log(result); + + this.callbackHandler = new PythonProcessCallbackHandler(this, this.stream, this._idDispenser); + this.callbackHandler.on("detach", () => this.emit("detach")); + this.callbackHandler.on("last", () => this.emit("last")); + this.callbackHandler.on("moduleLoaded", arg=> this.emit("moduleLoaded", arg)); + this.callbackHandler.on("threadCreated", arg=> this.emit("threadCreated", arg)); + this.callbackHandler.on("threadExited", arg=> this.emit("threadExited", arg)); + this.callbackHandler.on("stepCompleted", arg=> this.onPythonStepCompleted(arg)); + this.callbackHandler.on("breakpointSet", arg=> this.onBreakpointSet(arg, true)); + this.callbackHandler.on("breakpointNotSet", arg=> this.onBreakpointSet(arg, false)); + this.callbackHandler.on("output", (pyThread, output) => this.emit("output", pyThread, output)); + this.callbackHandler.on("exceptionRaised", (pyThread, ex, brkType) => { + this._lastExecutedThread = pyThread; + this.emit("exceptionRaised", pyThread, ex, brkType); + }); + this.callbackHandler.on("breakpointHit", (pyThread, breakpointId) => this.onBreakpointHit(pyThread, breakpointId)); + this.callbackHandler.on("processLoaded", arg=> { + this._mainThread = arg; + this._lastExecutedThread = this._mainThread; + this.emit("processLoaded", arg); + }); + this.callbackHandler.HandleIncomingData(); + } + + public HandleIncomingData(buffer: Buffer) { + this.stream.Append(buffer); + this.callbackHandler.HandleIncomingData(); + } + + //#region Step Commands + private stepCommands: IStepCommand[] = []; + private onPythonStepCompleted(pyThread: IPythonThread) { + this._lastExecutedThread = pyThread; + //Find the last step command associated with this threadCreated + var index = this.stepCommands.findIndex(cmd=> cmd.PythonThreadId === pyThread.Id); + if (index === -1) { + this.emit("error", "command.step.completed", `Uknown thread ${pyThread.Id}`); + //Hmm this is not possible, log this exception and carry on + return; + } + + var cmd = this.stepCommands.splice(index, 1)[0]; + cmd.PromiseResolve(pyThread); + } + private sendStepCommand(threadId: number, command: Buffer, doNotWaitForResponse: boolean = false): Promise { + return new Promise((resolve, reject) => { + var cmd: IStepCommand = { + PromiseResolve: resolve, + PythonThreadId: threadId + }; + + this.stepCommands.push(cmd); + this.stream.Write(command); + this.stream.WriteInt64(threadId); + + if (doNotWaitForResponse) { + if (this.Threads.has(threadId)) { + resolve(this.Threads.get(threadId)); + } + else { + resolve(); + } + } + }); + } + public SendStepOver(threadId: number): Promise { + return this.sendStepCommand(threadId, Commands.StepOverCommandBytes); + } + public SendStepOut(threadId: number) { + return this.sendStepCommand(threadId, Commands.StepOutCommandBytes); + } + public SendStepInto(threadId: number) { + return this.sendStepCommand(threadId, Commands.StepIntoCommandBytes); + } + //#endregion + private onBreakpointHit(pyThread: IPythonThread, breakpointId: number) { + this._lastExecutedThread = pyThread; + this.emit("breakpointHit", pyThread, breakpointId); + } + private onBreakpointSet(breakpointId: number, success: boolean) { + //Find the last breakpoint command associated with this breakpoint + var index = this.breakpointCommands.findIndex(cmd=> cmd.Id === breakpointId); + if (index === -1) { + //Hmm this is not possible, log this exception and carry on + this.emit("error", "command.breakpoint.hit", `Uknown Breakpoit Id ${breakpointId}`); + return; + } + + var cmd = this.breakpointCommands.splice(index, 1)[0]; + if (success) { + cmd.PromiseResolve(); + } + else { + cmd.PromiseReject(); + } + } + + private breakpointCommands: IBreakpointCommand[] = []; + public DisableBreakPoint(breakpoint: IPythonBreakpoint) { + if (breakpoint.IsDjangoBreakpoint) { + this.stream.Write(Commands.RemoveDjangoBreakPointCommandBytes); + } else { + this.stream.Write(Commands.RemoveBreakPointCommandBytes); + } + this.stream.WriteInt32(breakpoint.LineNo); + this.stream.WriteInt32(breakpoint.Id); + if (breakpoint.IsDjangoBreakpoint) { + // this.writeStringToSocket(breakpoint.Filename); + this.stream.WriteString(breakpoint.Filename); + } + } + + public BindBreakpoint(brkpoint: IPythonBreakpoint): Promise { + return new Promise((resolve, reject) => { + var bkCmd: IBreakpointCommand = { + Id: brkpoint.Id, + PromiseResolve: resolve, + PromiseReject: reject + }; + this.breakpointCommands.push(bkCmd); + + if (brkpoint.IsDjangoBreakpoint) { + this.stream.Write(Commands.AddDjangoBreakPointCommandBytes); + } + else { + this.stream.Write(Commands.SetBreakPointCommandBytes); + } + this.stream.WriteInt32(brkpoint.Id); + + this.stream.WriteInt32(brkpoint.LineNo); + + this.stream.WriteString(brkpoint.Filename); + + if (!brkpoint.IsDjangoBreakpoint) { + this.SendCondition(brkpoint); + this.SendPassCount(brkpoint); + } + }); + } + + private SendCondition(breakpoint: IPythonBreakpoint) { + this.stream.WriteInt32(breakpoint.ConditionKind); + this.stream.WriteString(breakpoint.Condition || ""); + } + private SendPassCount(breakpoint: IPythonBreakpoint) { + // DebugWriteCommand("Send BP pass count"); + this.stream.WriteInt32(breakpoint.PassCountKind); + this.stream.WriteInt32(breakpoint.PassCount); + } + + public SendResumeThread(threadId: number): Promise { + return this.sendStepCommand(threadId, Commands.ResumeThreadCommandBytes, true); + } + public SendContinue(): Promise { + return new Promise(resolve => { + this.stream.Write(Commands.ResumeAllCommandBytes); + resolve(); + }) + } + public AutoResumeThread(threadId: number) { + + } + public SendClearStepping(threadId: number) { + + } + + public ExecuteText(text: string, reprKind: PythonEvaluationResultReprKind, stackFrame: IPythonStackFrame): Promise { + return new Promise((resolve, reject) => { + var executeId = this._idDispenser.Allocate(); + var cmd: IExecutionCommand = { + Id: executeId, + Text: text, + Frame: stackFrame, + PromiseResolve: resolve, + PromiseReject: reject + }; + this.PendingExecuteCommands.set(executeId, cmd); + console.log(`ExecuteText for ${text} with Execute id ${executeId}`); + this.stream.Write(Commands.ExecuteTextCommandBytes); + this.stream.WriteString(text); + this.stream.WriteInt64(stackFrame.Thread.Id); + this.stream.WriteInt32(stackFrame.FrameId); + this.stream.WriteInt32(executeId); + this.stream.WriteInt32(stackFrame.Kind); + this.stream.WriteInt32(reprKind); + }); + } + + public EnumChildren(text: string, stackFrame: IPythonStackFrame, timeout: number): Promise { + return new Promise((resolve, reject) => { + var executeId = this._idDispenser.Allocate(); + if (typeof (executeId) !== "number") { + var y = ""; + } + var cmd: IChildEnumCommand = { + Id: executeId, + Frame: stackFrame, + PromiseResolve: resolve, + PromiseReject: reject + }; + console.log(`EnumChildren for ${text} with Execute id ${executeId}`); + this.PendingChildEnumCommands.set(executeId, cmd); + setTimeout(() => { + if (this.PendingChildEnumCommands.has(executeId)) { + this.PendingChildEnumCommands.delete(executeId); + } + var seconds = timeout / 1000; + reject(`Enumerating children for ${text} timed out after ${seconds} seconds.`); + }, timeout); + + this.stream.Write(Commands.GetChildrenCommandBytes); + this.stream.WriteString(text); + this.stream.WriteInt64(stackFrame.Thread.Id); + this.stream.WriteInt32(stackFrame.FrameId); + this.stream.WriteInt32(executeId); + this.stream.WriteInt32(stackFrame.Kind); + }); + } + public SetLineNumber(pythonStackFrame: IPythonStackFrame, lineNo: number) { + + } + +} diff --git a/src/client/debugger/vs/PythonProcessCallbackHandler.ts b/src/client/debugger/vs/PythonProcessCallbackHandler.ts new file mode 100644 index 000000000000..a6cdc076dab3 --- /dev/null +++ b/src/client/debugger/vs/PythonProcessCallbackHandler.ts @@ -0,0 +1,456 @@ +import {FrameKind, IPythonProcess, IPythonThread, IPythonModule, IPythonEvaluationResult, IPythonStackFrame} from './Common/Contracts'; +import {IDjangoStackFrame, PythonEvaluationResultFlags, PythonLanguageVersion, IChildEnumCommand, IPythonException, IExecutionCommand} from './Common/Contracts'; +import * as utils from './Common/Utils'; +import {EventEmitter} from 'events'; +import {Commands} from './ProxyCommands'; +import {SocketStream} from './Common/SocketStream'; + +export class PythonProcessCallbackHandler extends EventEmitter { + private process: IPythonProcess; + private idDispenser: utils.IdDispenser; + private stream: SocketStream; + private _stoppedForException: boolean; + constructor(process: IPythonProcess, stream: SocketStream, idDispenser: utils.IdDispenser) { + super(); + this.process = process; + this.stream = stream; + this.idDispenser = idDispenser; + } + + public HandleIncomingData() { + this.stream.BeginTransaction(); + + var cmd = this.stream.ReadAsciiString(4); + if (this.stream.HasInsufficientDataForReading) { + return; + } + + switch (cmd) { + case "MODL": this.HandleModuleLoad(); break; + case "LOAD": this.HandleProcessLoad(); break; + case "STPD": this.HandleStepDone(); break; + case "NEWT": this.HandleThreadCreate(); break; + case "EXTT": this.HandleThreadExit(); break; + case "THRF": this.HandleThreadFrameList(); break; + case "OUTP": this.HandleDebuggerOutput(); break; + case "BRKS": this.HandleBreakPointSet(); break; + case "BRKF": this.HandleBreakPointFailed(); break; + case "BRKH": this.HandleBreakPointHit(); break; + case "DETC": this.HandleDetach(); break; // detach, report process exit + case "LAST": this.HandleLast(); break; + case "CHLD": this.HandleEnumChildren(); break; + case "REQH": this.HandleRequestHandlers(); break; + case "EXCP": this.HandleException(); break; + case "EXCR": this.HandleExecutionResult(); break; + case "EXCE": this.HandleExecutionException(); break; + default: { + var x = ""; + } + } + + if (this.stream.HasInsufficientDataForReading) { + //Most possibly due to insufficient data + this.stream.RollBackTransaction(); + return; + } + + this.stream.EndTransaction(); + if (this.stream.Length > 0) { + this.HandleIncomingData(); + } + } + + private get LanguageVersion(): PythonLanguageVersion { + return PythonLanguageVersion.Is2; + } + private HandleDetach() { + this.emit("detach"); + } + private HandleLast() { + this.emit("last"); + } + private HandleModuleLoad() { + var moduleId = this.stream.ReadInt32(); + var filename = this.stream.ReadString(); + + if (this.stream.HasInsufficientDataForReading) { + return; + } + + if (filename != null) { + // console.log(`Module Loaded ${moduleId}: ${filename}`); + this.emit("moduleLoaded", utils.CreatePythonModule(moduleId, filename)); + } + } + + private HandleDebuggerOutput() { + var threadId = this.stream.ReadInt64(); + var output = this.stream.ReadString(); + if (this.stream.HasInsufficientDataForReading) { + return; + } + + var pyThread: IPythonThread; + if (this.process.Threads.has(threadId)) { + pyThread = this.process.Threads.get(threadId); + } + this.emit("output", pyThread, output); + } + + private _createdFirstThread: boolean; + private HandleThreadCreate() { + var threadId = this.stream.ReadInt64(); + if (this.stream.HasInsufficientDataForReading) { + return; + } + + var pyThread = utils.CreatePythonThread(threadId, this._createdFirstThread, this.process); + this._createdFirstThread = true; + this.process.Threads.set(threadId, pyThread); + this.emit("threadCreated", pyThread); + } + + private HandleThreadExit() { + var threadId = this.stream.ReadInt64(); + if (this.stream.HasInsufficientDataForReading) { + return; + } + + var thread: IPythonThread; + if (this.process.Threads.has(threadId)) { + thread = this.process.Threads.get(threadId); + this.emit("threadExited", thread); + // this.process.Threads.delete(threadId); + } + } + + private HandleProcessLoad() { + var threadId = this.stream.ReadInt64(); + if (this.stream.HasInsufficientDataForReading) { + return; + } + var pyThread: IPythonThread; + if (this.process.Threads.has(threadId)) { + pyThread = this.process.Threads.get(threadId); + } + this.emit("processLoaded", pyThread); + console.log("Process loaded"); + } + + private HandleStepDone() { + var threadId = this.stream.ReadInt64(); + if (this.stream.HasInsufficientDataForReading) { + return; + } + var pyThread: IPythonThread; + if (this.process.Threads.has(threadId)) { + pyThread = this.process.Threads.get(threadId); + } + this.emit("stepCompleted", pyThread); + } + private HandleBreakPointFailed() { + var id = this.stream.ReadInt32(); + if (this.stream.HasInsufficientDataForReading) { + return; + } + this.emit("breakpointNotSet", id); + } + private HandleBreakPointSet() { + var id = this.stream.ReadInt32(); + if (this.stream.HasInsufficientDataForReading) { + return; + } + this.emit("breakpointSet", id); + } + private HandleBreakPointHit() { + var breakId = this.stream.ReadInt32(); + var threadId = this.stream.ReadInt64(); + if (this.stream.HasInsufficientDataForReading) { + return; + } + + var pyThread: IPythonThread; + if (this.process.Threads.has(threadId)) { + pyThread = this.process.Threads.get(threadId); + } + this.emit("breakpointHit", pyThread, breakId); + } + private HandleRequestHandlers() { + var filename = this.stream.ReadString(); + if (this.stream.HasInsufficientDataForReading) { + return; + } + var y = ""; + // this.emit("error", filename); + console.error("Exception handlers requested for: " + filename); + // var statements = GetHandledExceptionRanges(filename); + + this.stream.Write(Commands.SetExceptionHandlerInfoCommandBytes); + this.stream.WriteString(filename); + + this.stream.WriteInt32(0); + } + private HandleException() { + var typeName = this.stream.ReadString(); + var threadId = this.stream.ReadInt64(); + var breakType = this.stream.ReadInt32(); + var desc = this.stream.ReadString(); + if (this.stream.HasInsufficientDataForReading) { + return; + } + + if (typeName != null && desc != null) { + console.error("Exception: " + desc); + var ex: IPythonException = { + TypeName: typeName, + Description: desc + }; + var pyThread: IPythonThread; + if (this.process.Threads.has(threadId)) { + pyThread = this.process.Threads.get(threadId); + } + this.emit("exceptionRaised", pyThread, ex, breakType == 1 /* BREAK_TYPE_UNHANLDED */); + } + this._stoppedForException = true; + } + private HandleExecutionException() { + var execId = this.stream.ReadInt32(); + var exceptionText = this.stream.ReadString(); + if (this.stream.HasInsufficientDataForReading) { + return; + } + + var cmd: IExecutionCommand = null; + if (this.process.PendingExecuteCommands.has(execId)) { + cmd = this.process.PendingExecuteCommands.get(execId); + if (this.process.PendingExecuteCommands.has(execId)) { + this.process.PendingExecuteCommands.delete(execId); + } + cmd.PromiseReject(exceptionText); + console.log(`ExecuteText Exception ${execId}`); + } else { + console.error("Received execution result with unknown execution ID " + execId); + } + this.idDispenser.Free(execId); + } + private HandleExecutionResult() { + var execId = this.stream.ReadInt32(); + if (this.stream.HasInsufficientDataForReading) { + return; + } + + var cmd: IExecutionCommand = null; + if (this.process.PendingExecuteCommands.has(execId)) { + cmd = this.process.PendingExecuteCommands.get(execId); + } else { + console.error("Received execution result with unknown execution ID " + execId); + } + + if (cmd === null) { + // Passing null for parameters other than stream is okay as long + // as we drop the result. + this.ReadPythonObject(null, null, null); + if (this.stream.HasInsufficientDataForReading) { + return; + } + } + else { + var evalResult = this.ReadPythonObject(cmd.Text, null, cmd.Frame); + if (this.stream.HasInsufficientDataForReading) { + return; + } + + cmd.PromiseResolve(evalResult); + } + + if (cmd != null) { + if (this.process.PendingExecuteCommands.has(execId)) { + this.process.PendingExecuteCommands.delete(execId); + } + console.log(`ExecuteText Result ${execId}`); + } + this.idDispenser.Free(execId); + } + + private HandleEnumChildren() { + var execId = this.stream.ReadInt32(); + if (this.stream.HasInsufficientDataForReading) { + return; + } + + var cmd: IChildEnumCommand = null; + if (this.process.PendingChildEnumCommands.has(execId)) { + cmd = this.process.PendingChildEnumCommands.get(execId); + } else { + console.error("Received enum children result with unknown execution ID " + execId); + } + + var childrenCount = this.stream.ReadInt32(); + if (this.stream.HasInsufficientDataForReading) { + return; + } + + const children: IPythonEvaluationResult[] = []; + for (var childCount = 0; childCount < childrenCount; childCount++) { + const childName = this.stream.ReadString(); + const childExpr = this.stream.ReadString(); + if (this.stream.HasInsufficientDataForReading) { + return; + } + + var obj = this.ReadPythonObject(childExpr, childName, cmd === null ? null : cmd.Frame); + if (this.stream.HasInsufficientDataForReading) { + return; + } + children.push(obj); + } + + if (cmd != null) { + cmd.PromiseResolve(children); + if (this.process.PendingChildEnumCommands.has(execId)) { + this.process.PendingChildEnumCommands.delete(execId); + } + console.log(`ExecuteText Result with Execute id ${execId}`); + } + this.idDispenser.Free(execId); + } + private HandleThreadFrameList() { + var frames: IPythonStackFrame[] = []; + var threadId = this.stream.ReadInt64(); + if (this.stream.HasInsufficientDataForReading) { + return; + } + + var pyThread: IPythonThread; + if (this.process.Threads.has(threadId)) { + pyThread = this.process.Threads.get(threadId); + } + + var threadName = this.stream.ReadString(); + var frameCount = this.stream.ReadInt32(); + if (this.stream.HasInsufficientDataForReading) { + return; + } + + for (var i = 0; i < frameCount; i++) { + var startLine = this.stream.ReadInt32(); + var endLine = this.stream.ReadInt32(); + var lineNo = this.stream.ReadInt32(); + var frameName = this.stream.ReadString(); + var filename = this.stream.ReadString(); + var argCount = this.stream.ReadInt32(); + var frameKind = this.stream.ReadInt32(); + if (this.stream.HasInsufficientDataForReading) { + return; + } + + var frame: IPythonStackFrame = null; + if (pyThread != null) { + switch (frameKind) { + case FrameKind.Django: { + var sourceFile = this.stream.ReadString(); + var sourceLine = this.stream.ReadInt32(); + if (this.stream.HasInsufficientDataForReading) { + return; + } + + var djangoFrame: IDjangoStackFrame = { + EndLine: endLine, FileName: filename, + FrameId: i, FunctionName: frameName, + Kind: frameKind, LineNo: lineNo, + Locals: [], Parameters: [], + Thread: pyThread, SourceFile: sourceFile, + SourceLine: sourceLine, StartLine: startLine + }; + + frame = djangoFrame; + break; + } + default: { + frame = { + EndLine: endLine, FileName: filename, + FrameId: i, FunctionName: frameName, + Kind: frameKind, LineNo: lineNo, + Locals: [], Parameters: [], + Thread: pyThread, StartLine: startLine + }; + break; + } + } + + } + + var varCount = this.stream.ReadInt32(); + if (this.stream.HasInsufficientDataForReading) { + return; + } + + var variables: IPythonEvaluationResult[] = []; + for (var j = 0; j < varCount; j++) { + var name = this.stream.ReadString(); + if (this.stream.HasInsufficientDataForReading) { + return; + } + + if (frame != null) { + var variableObj = this.ReadPythonObject(name, name, frame); + if (this.stream.HasInsufficientDataForReading) { + return; + } + + variables.push(variableObj); + } + } + if (frame != null) { + frame.Parameters = variables.splice(0, argCount); + frame.Locals = variables; + frames.push(frame); + } + } + + if (pyThread != null) { + pyThread.Frames = frames; + if (typeof threadName === "string" && threadName.length > 0) { + pyThread.Name = threadName; + } + } + } + + private ReadPythonObject(expr: string, childName: string, frame: IPythonStackFrame): IPythonEvaluationResult { + var objRepr = this.stream.ReadString(); + var hexRepr = this.stream.ReadString(); + var typeName = this.stream.ReadString(); + var length = this.stream.ReadInt64(); + var flags = this.stream.ReadInt32(); + + if (this.stream.HasInsufficientDataForReading) { + return; + } + + if ((flags & PythonEvaluationResultFlags.Raw) == 0 && ((typeName === "unicode" && this.LanguageVersion === PythonLanguageVersion.Is2) + || (typeName === "str" && this.LanguageVersion === PythonLanguageVersion.Is3))) { + objRepr = utils.FixupEscapedUnicodeChars(objRepr); + } + + if (typeName == "bool") { + hexRepr = null; + } + + var pythonEvaluationResult: IPythonEvaluationResult = { + ChildName: childName, + Process: this.process, + IsExpandable: (flags & PythonEvaluationResultFlags.Expandable) > 0, + Flags: flags, + StringRepr: objRepr, + HexRepr: hexRepr, + TypeName: typeName, + Expression: expr, + Length: length, + Frame: frame + }; + + return pythonEvaluationResult; + } + + +} diff --git a/src/client/debugger/vs/VSDebugger.ts b/src/client/debugger/vs/VSDebugger.ts new file mode 100644 index 000000000000..84c975a354ee --- /dev/null +++ b/src/client/debugger/vs/VSDebugger.ts @@ -0,0 +1,510 @@ +"use string"; + +import {Variable, DebugSession, InitializedEvent, TerminatedEvent, StoppedEvent, OutputEvent, Thread, StackFrame, Scope, Source, Handles} from 'vscode-debugadapter'; +import {ThreadEvent} from 'vscode-debugadapter'; +import {DebugProtocol} from 'vscode-debugprotocol'; +import {readFileSync} from 'fs'; +import {basename} from 'path'; +import * as path from 'path'; +import * as os from 'os'; +import * as fs from 'fs'; +import * as child_process from 'child_process'; +import * as StringDecoder from 'string_decoder'; +import * as net from 'net'; +import {PythonProcess} from './PythonProcess'; +import {FrameKind, IPythonProcess, IPythonThread, IPythonModule, IPythonEvaluationResult, IPythonStackFrame, IDebugServer} from './Common/Contracts'; +import {IPythonBreakpoint, PythonBreakpointConditionKind, PythonBreakpointPassCountKind, IPythonException, PythonEvaluationResultReprKind} from './Common/Contracts'; + +const CHILD_ENUMEARATION_TIMEOUT = 5000; + +/** + * This interface should always match the schema found in the mock-debug extension manifest. + */ +export interface LaunchRequestArguments { + /** An absolute path to the program to debug. */ + program: string; + /** Automatically stop target after launch. If not specified, target does not stop. */ + stopOnEntry?: boolean; + args: string[]; + pythonPath: string; +} + +interface IDebugVariable { + variables: IPythonEvaluationResult[]; + evaluateChildren?: Boolean; +} + +class PythonDebugSession extends DebugSession { + // we don't support multiple threads, so we can use a hardcoded ID for the default thread + private static THREAD_ID = 1; + private _variableHandles: Handles; + private breakPointCounter: number = 0; + private registeredBreakpoints: Map; + private registeredBreakpointsByFileName: Map; + + private debuggerLoaded: Promise; + private debuggerLoadedPromiseResolve: () => void; + public constructor(debuggerLinesStartAt1: boolean, isServer: boolean) { + super(debuggerLinesStartAt1, isServer === true); + this._variableHandles = new Handles(); + this.registeredBreakpoints = new Map(); + this.registeredBreakpointsByFileName = new Map(); + } + + protected initializeRequest(response: DebugProtocol.InitializeResponse, args: DebugProtocol.InitializeRequestArguments): void { + this.sendResponse(response); + + // now we are ready to accept breakpoints -> fire the initialized event to give UI a chance to set breakpoints + this.sendEvent(new InitializedEvent()); + } + private startedHandlingMessages:Boolean; + private pythonProcess: PythonProcess; + private debugSocketServer: net.Server; + private startDebugServer(): Promise { + return new Promise((resolve, reject) => { + var that = this; + this.pythonProcess = new PythonProcess(0, ""); + this.InitializeEventHandlers(); + this.debugSocketServer = net.createServer(c => { //'connection' listener + var pythonProcess: PythonProcess; + var connected = false; + console.log('client connected'); + c.on('end', (ex) => { + var msg = "Debugger client disconneced, " + ex; + that.sendEvent(new OutputEvent(msg + "\n", "stderr")); + console.log(msg); + }); + c.on("data", (buffer: Buffer) => { + if (!connected) { + connected = true; + that.pythonProcess.Connect(buffer, c); + } + else { + that.pythonProcess.HandleIncomingData(buffer) + that.startedHandlingMessages = true; + } + }); + c.on("close", d=> { + var msg = "Debugger client closed, " + d; + console.log(msg); + that.emit("detach", d); + that.onDetachDebugger(); + }); + c.on("error", d=> { + // var msg = "Debugger client error, " + d; + // that.sendEvent(new OutputEvent(msg + "\n", "Python")); + // console.log(msg); + // // that.onDetachDebugger(); + }); + c.on("timeout", d=> { + var msg = "Debugger client timedout, " + d; + that.sendEvent(new OutputEvent(msg + "\n", "stderr")); + console.log(msg); + }); + }); + this.debugSocketServer.on("error", ex=> { + var exMessage = JSON.stringify(ex); + var msg = ""; + if (ex.code === "EADDRINUSE") { + msg = `The port used for debugging is in use, please try again or try restarting Visual Studio Code, Error = ${exMessage}`; + } + else { + msg = `There was an error in starting the debug server. Error = ${exMessage}`; + } + that.sendEvent(new OutputEvent(msg + "\n", "stderr")); + console.log(msg); + reject(msg); + }); + + this.debugSocketServer.listen(0, () => { + var server = that.debugSocketServer.address(); + console.log(`Debug server started, listening on port ${server.port}`); + resolve({ port: server.port }); + }); + }); + } + + private stopDebugServer() { + try { + this.debugSocketServer.close(); + } + catch (ex) { + } + try { + this.pyProc.kill(); + } + catch (ex) { + } + this.debugSocketServer = null; + this.pyProc = null; + } + + private InitializeEventHandlers() { + this.pythonProcess.on("threadExited", arg => this.onPythonThreadExited(arg)); + this.pythonProcess.on("moduleLoaded", arg=> this.onPythonModuleLoaded(arg)); + this.pythonProcess.on("threadCreated", arg=> this.onPythonThreadCreated(arg)); + this.pythonProcess.on("processLoaded", arg=> this.onPythonProcessLoaded(arg)); + this.pythonProcess.on("output", (pyThread, output) => this.onDebuggerOutput(pyThread, output)); + this.pythonProcess.on("exceptionRaised", (pyThread, ex) => this.onPythonException(pyThread, ex)); + this.pythonProcess.on("breakpointHit", (pyThread, breakpointId) => this.onBreakpointHit(pyThread, breakpointId)); + this.pythonProcess.on("detach", () => this.onDetachDebugger()); + } + private onDetachDebugger() { + this.stopDebugServer(); + this.sendEvent(new TerminatedEvent()); + this.shutdown(); + } + private onPythonThreadCreated(pyThread: IPythonThread) { + this.sendEvent(new ThreadEvent("started", pyThread.Id)); + } + private onPythonException(pyThread: IPythonThread, ex: IPythonException) { + this.sendEvent(new StoppedEvent("exception", pyThread.Id, `${ex.TypeName}, ${ex.Description}`)); + this.sendEvent(new OutputEvent(`${ex.TypeName}, ${ex.Description}\n`, "stderr")); + // this.sendEvent(new StoppedEvent("breakpoint", pyThread.Id)); + } + private onPythonThreadExited(pyThread: IPythonThread) { + this.sendEvent(new ThreadEvent("exited", pyThread.Id)); + } + private onPythonModuleLoaded(module: IPythonModule) { + } + private onPythonProcessLoaded(pyThread: IPythonThread) { + this.sendResponse(this.entryResponse); + this.sendEvent(new StoppedEvent("entry", pyThread.Id)); + this.debuggerLoadedPromiseResolve(); + } + private onDebuggerOutput(pyThread: IPythonThread, output: string) { + this.sendEvent(new OutputEvent(output)); + } + + private breakPointsLoaded: Promise; + private breakPointsResolve: () => void; + private entryResponse: DebugProtocol.LaunchResponse; + protected launchRequest(response: DebugProtocol.LaunchResponse, args: LaunchRequestArguments): void { + var fileDir = path.dirname(args.program); + var fileNameWithoutPath = path.basename(args.program); + + var pythonPath = "python"; + if (typeof args.pythonPath === "string" && args.pythonPath.trim().length > 0) { + pythonPath = args.pythonPath; + } + + this.debuggerLoaded = new Promise(resolve=> { + this.debuggerLoadedPromiseResolve = resolve; + }); + this.entryResponse = response; + + var that = this; + this.startDebugServer().then(dbgServer => { + dbgServer.port + var vsDebugOptions = "RedirectOutput";//,\"WaitOnNormalExit\""; + //GUID is hardcoded for now, will have to be fixed + var currentFileName = module.filename; + var ptVSToolsFilePath = path.join(path.dirname(currentFileName), "..", "..", "..", "..", "pythonFiles", "PythonTools", "visualstudio_py_launcher.py");// ""; //C:\Users\djayamanne\.vscode\extensions\pythonVSCode\pythonFiles\PythonTools + var programArgs = Array.isArray(args.args) && args.args.length > 0 ? args.args.join(" ") : ""; + + var commandLine = `${pythonPath} \"${ptVSToolsFilePath}\" \"${fileDir}" ${dbgServer.port} 34806ad9-833a-4524-8cd6-18ca4aa74f14 \"${vsDebugOptions}\" ${fileNameWithoutPath}`; + that.pyProc = child_process.exec(commandLine, { cwd: fileDir }, (error, stdout, stderr) => { + if (that.startedHandlingMessages){ + return; + } + var hasErrors = (error && error.message.length > 0) || (stderr && stderr.length > 0); + if (hasErrors && (typeof stdout !== "string" || stdout.length === 0)) { + var errorMsg = (error && error.message) ? error.message : (stderr && stderr.length > 0 ? stderr.toString("utf-8") : ""); + that.sendEvent(new OutputEvent(errorMsg + "\n", "stderr")); + that.sendErrorResponse(that.entryResponse, 2000, errorMsg); + console.error(errorMsg); + } + + //resolve(stdout.toString('utf-8')); + }); + }); + } + private pyProc: child_process.ChildProcess; + private onBreakpointHit(pyThread: IPythonThread, breakpointId: number) { + if (this.registeredBreakpoints.has(breakpointId)) { + this.sendEvent(new StoppedEvent("breakpoint", pyThread.Id)); + } + else { + this.pythonProcess.SendResumeThread(pyThread.Id); + } + } + protected setBreakPointsRequest(response: DebugProtocol.SetBreakpointsResponse, args: DebugProtocol.SetBreakpointsArguments): void { + this.debuggerLoaded.then(() => { + if (!this.registeredBreakpointsByFileName.has(args.source.path)) { + this.registeredBreakpointsByFileName.set(args.source.path, []); + } + + var breakpoints: { verified: boolean, line: number }[] = []; + var breakpointsToRemove = []; + var linesToAdd: number[] = []; + if (Array.isArray(args.breakpoints)) { + linesToAdd = args.breakpoints.map(b=> b.line); + } + else { + linesToAdd = args.lines; + } + + var linesToRemove = []; + var registeredBks = this.registeredBreakpointsByFileName.get(args.source.path); + linesToRemove = registeredBks.map(b=> b.LineNo).filter(oldLine=> linesToAdd.indexOf(oldLine) === -1); + + var linesToAddPromises = linesToAdd.map(line=> { + return new Promise(resolve=> { + var breakpoint: IPythonBreakpoint = { + Condition: "", + ConditionKind: PythonBreakpointConditionKind.Always, + Filename: args.source.path, + Id: this.breakPointCounter++, + LineNo: line, + PassCount: 0, + PassCountKind: PythonBreakpointPassCountKind.Always + }; + + this.pythonProcess.BindBreakpoint(breakpoint).then(() => { + this.registeredBreakpoints.set(breakpoint.Id, breakpoint); + breakpoints.push({ verified: true, line: line }); + registeredBks.push(breakpoint); + resolve(); + }, reason=> { + this.registeredBreakpoints.set(breakpoint.Id, breakpoint); + breakpoints.push({ verified: false, line: line }); + resolve(); + }); + }); + }); + + var linesToRemovePromises = linesToRemove.map(line=> { + return new Promise(resolve=> { + var registeredBks = this.registeredBreakpointsByFileName.get(args.source.path); + var bk = registeredBks.filter(b=> b.LineNo === line)[0]; + this.pythonProcess.DisableBreakPoint(bk); + }); + }); + + var promises = linesToAddPromises.concat(linesToRemovePromises); + Promise.all(promises).then(() => { + response.body = { + breakpoints: breakpoints + }; + + this.sendResponse(response); + }); + }); + } + + protected threadsRequest(response: DebugProtocol.ThreadsResponse): void { + // return the default thread + var threads = []; + this.pythonProcess.Threads.forEach(t=> { + threads.push(new Thread(t.Id, t.Name)); + }); + + response.body = { + threads: threads + }; + this.sendResponse(response); + } + + private currentStackThread: IPythonThread; + protected stackTraceRequest(response: DebugProtocol.StackTraceResponse, args: DebugProtocol.StackTraceArguments): void { + if (!this.pythonProcess.Threads.has(args.threadId)) { + response.body = { + stackFrames: [] + }; + this.currentStackThread = null; + this.sendResponse(response); + } + + var pyThread = this.pythonProcess.Threads.get(args.threadId); + this.currentStackThread = pyThread; + var maxFrames = typeof args.levels === "number" && args.levels > 0 ? args.levels : pyThread.Frames.length - 1; + maxFrames = maxFrames < pyThread.Frames.length ? maxFrames : pyThread.Frames.length; + + var frames = []; + for (var counter = 0; counter < maxFrames; counter++) { + var frame = pyThread.Frames[counter]; + frames.push(new StackFrame(counter, frame.FunctionName, + new Source(path.basename(frame.FileName), this.convertDebuggerPathToClient(frame.FileName)), + this.convertDebuggerLineToClient(frame.LineNo - 1), + 0)); + } + + response.body = { + stackFrames: frames + }; + + this.sendResponse(response); + } + + protected stepInRequest(response: DebugProtocol.StepInResponse): void { + this.pythonProcess.SendStepInto(this.pythonProcess.LastExecutedThread.Id).then(pyThread => { + this.sendResponse(response); + this.sendEvent(new StoppedEvent("step", pyThread.Id)); + }); + } + + protected stepOutRequest(response: DebugProtocol.StepInResponse): void { + this.pythonProcess.SendStepOut(this.pythonProcess.LastExecutedThread.Id).then(pyThread => { + this.sendResponse(response); + this.sendEvent(new StoppedEvent("return", pyThread.Id)); + }); + } + + protected continueRequest(response: DebugProtocol.ContinueResponse, args: DebugProtocol.ContinueArguments): void { + this.pythonProcess.SendContinue().then(() => { + this.sendResponse(response); + }); + } + + protected nextRequest(response: DebugProtocol.NextResponse, args: DebugProtocol.NextArguments): void { + this.pythonProcess.SendStepOver(this.pythonProcess.LastExecutedThread.Id).then(pyThread => { + this.sendResponse(response); + this.sendEvent(new StoppedEvent("next", pyThread.Id)); + }); + } + + protected evaluateRequest(response: DebugProtocol.EvaluateResponse, args: DebugProtocol.EvaluateArguments): void { + //Find the last thread for which we dislay the stack frames + if (!this.currentStackThread) { + response.body = { + result: null, + variablesReference: 0 + }; + return this.sendResponse(response); + } + + var frame = this.currentStackThread.Frames[args.frameId]; + this.pythonProcess.ExecuteText(args.expression, PythonEvaluationResultReprKind.Normal, frame).then(result=> { + let variablesReference = 0; + //If this value can be expanded, then create a vars ref for user to expand it + if (result.IsExpandable) { + const parentVariable: IDebugVariable = { + variables: [result], + evaluateChildren: true + }; + variablesReference = this._variableHandles.create(parentVariable); + } + + response.body = { + result: result.StringRepr, + variablesReference: variablesReference + }; + this.sendResponse(response); + }, + error => { + // this.sendResponse(response); + this.sendErrorResponse(response, 2000, error); + } + ); + } + + + protected scopesRequest(response: DebugProtocol.ScopesResponse, args: DebugProtocol.ScopesArguments): void { + //Find the last thread for which we dislay the stack frames + if (!this.currentStackThread) { + response.body = { + scopes: [] + }; + return this.sendResponse(response); + } + + var frame = this.currentStackThread.Frames[args.frameId]; + frame.Locals + var scopes = []; + if (Array.isArray(frame.Locals) && frame.Locals.length > 0) { + let values: IDebugVariable = { variables: frame.Locals }; + scopes.push(new Scope("Local", this._variableHandles.create(values), false)); + } + if (Array.isArray(frame.Parameters) && frame.Parameters.length > 0) { + let values: IDebugVariable = { variables: frame.Parameters }; + scopes.push(new Scope("Arguments", this._variableHandles.create(values), false)); + } + response.body = { scopes }; + this.sendResponse(response); + } + + protected variablesRequest(response: DebugProtocol.VariablesResponse, args: DebugProtocol.VariablesArguments): void { + var varRef = this._variableHandles.get(args.variablesReference); + + if (varRef.evaluateChildren !== true) { + let variables = []; + varRef.variables.forEach(variable=> { + let variablesReference = 0; + //If this value can be expanded, then create a vars ref for user to expand it + if (variable.IsExpandable) { + const parentVariable: IDebugVariable = { + variables: [variable], + evaluateChildren: true + }; + variablesReference = this._variableHandles.create(parentVariable); + } + + variables.push({ + name: variable.Expression, + value: variable.StringRepr, + variablesReference: variablesReference + }); + }); + + response.body = { + variables: variables + }; + + return this.sendResponse(response); + } + + + + //Ok, we need to evaluate the children of the current variable + var variables = []; + var promises = varRef.variables.map(variable=> { + return variable.Process.EnumChildren(variable.Expression, variable.Frame, CHILD_ENUMEARATION_TIMEOUT).then(children=> { + children.forEach(child=> { + let variablesReference = 0; + //If this value can be expanded, then create a vars ref for user to expand it + if (child.IsExpandable) { + const childVariable: IDebugVariable = { + variables: [child], + evaluateChildren: true + }; + variablesReference = this._variableHandles.create(childVariable); + } + + variables.push({ + name: child.ChildName, + value: child.StringRepr, + variablesReference: variablesReference + }); + }); + }, error=> { + this.sendErrorResponse(response, 2001, error); + }); + }); + + Promise.all(promises).then(() => { + response.body = { + variables: variables + }; + + return this.sendResponse(response); + }); + } + + //Unsupported features + protected pauseRequest(response: DebugProtocol.PauseResponse): void { + console.error('Not yet implemented: pauseRequest'); + this.sendErrorResponse(response, 2000, "Pause is not yet supported"); + } + + protected setExceptionBreakPointsRequest(response: DebugProtocol.SetExceptionBreakpointsResponse, args: DebugProtocol.SetExceptionBreakpointsArguments): void { + console.error('Not yet implemented: setExceptionBreakPointsRequest'); + this.sendErrorResponse(response, 2000, "ExceptionBreakPointsRequest is not yet supported"); + } + + protected disconnectRequest(response: DebugProtocol.DisconnectResponse, args: DebugProtocol.DisconnectArguments) { + this.stopDebugServer(); + this.sendResponse(response); + } +} + +DebugSession.run(PythonDebugSession); diff --git a/src/client/formatters/baseFormatter.ts b/src/client/formatters/baseFormatter.ts index 3ab3256ebd75..3e850243d323 100644 --- a/src/client/formatters/baseFormatter.ts +++ b/src/client/formatters/baseFormatter.ts @@ -24,28 +24,30 @@ export abstract class BaseFormatter { protected provideDocumentFormattingEdits(document: vscode.TextDocument, options: vscode.FormattingOptions, token: vscode.CancellationToken, cmdLine: string): Thenable { var fileDir = path.dirname(document.uri.fsPath); return new Promise((resolve, reject) => { - var filePath = document.uri.fsPath; - if (!fs.existsSync(filePath)) { - vscode.window.showErrorMessage(`File ${filePath} does not exist`) - return resolve([]); - } - - this.outputChannel.clear(); - - sendCommand(cmdLine, fileDir).then(data=> { - var formattedText = data; - if (document.getText() === formattedText) { + document.save().then(saved=> { + var filePath = document.uri.fsPath; + if (!fs.existsSync(filePath)) { + vscode.window.showErrorMessage(`File ${filePath} does not exist`) return resolve([]); } - var range = new vscode.Range(document.lineAt(0).range.start, document.lineAt(document.lineCount - 1).range.end) - var txtEdit = new vscode.TextEdit(range, formattedText); - resolve([txtEdit]); + this.outputChannel.clear(); + + sendCommand(cmdLine, fileDir).then(data=> { + var formattedText = data; + if (document.getText() === formattedText) { + return resolve([]); + } - }, errorMsg => { - vscode.window.showErrorMessage(`There was an error in formatting the document. View the Python output window for details.`); - this.outputChannel.appendLine(errorMsg); - return resolve([]); + var range = new vscode.Range(document.lineAt(0).range.start, document.lineAt(document.lineCount - 1).range.end) + var txtEdit = new vscode.TextEdit(range, formattedText); + resolve([txtEdit]); + + }, errorMsg => { + vscode.window.showErrorMessage(`There was an error in formatting the document. View the Python output window for details.`); + this.outputChannel.appendLine(errorMsg); + return resolve([]); + }); }); }); } diff --git a/src/client/package.json b/src/client/package.json index 345a6612d5bd..4bf1f7bdafac 100644 --- a/src/client/package.json +++ b/src/client/package.json @@ -1,102 +1,156 @@ { - "name": "python", - "displayName": "Python", - "description": "Python auto complete and linting", - "version": "0.0.1", - "publisher": "donjayamanne", - "license": "SEE LICENSE IN LICENSE or README.MD", - "homepage": "https://github.com/DonJayamanne/pythonVSCode/blob/master/README.md", - "repository": { - "type": "git", - "url": "https://github.com/DonJayamanne/pythonVSCode" - }, - "bugs": { - "url": "https://github.com/DonJayamanne/pythonVSCode/issues" - }, - "icon": "images/icon.png", - "galleryBanner": { - "color": "#0000FF", - "theme": "dark" - }, - "engines": { - "vscode": "^0.10.1" - }, - "categories": [ - "Languages", - "Linters" - ], - "activationEvents": [ - "onLanguage:python", - "onLanguage:plaintext" - ], - "main": "../../out/client/extension", - "contributes": { - "debuggers": [ - { - "type": "python", - "label": "Python", - "enableBreakpointsFor": { - "languageIds": [ - "python" - ] - }, - "program": "./out/client/debugger/main.js", - "runtime": "node", - "configurationAttributes": { - "launch": { - "required": [ - "program" - ], + "name": "python", + "displayName": "Python", + "description": "Python auto complete and linting", + "version": "0.0.1", + "publisher": "donjayamanne", + "license": "SEE LICENSE IN LICENSE or README.MD", + "homepage": "https://github.com/DonJayamanne/pythonVSCode/blob/master/README.md", + "repository": { + "type": "git", + "url": "https://github.com/DonJayamanne/pythonVSCode" + }, + "bugs": { + "url": "https://github.com/DonJayamanne/pythonVSCode/issues" + }, + "icon": "images/icon.png", + "galleryBanner": { + "color": "#0000FF", + "theme": "dark" + }, + "engines": { + "vscode": "^0.10.1" + }, + "categories": [ + "Languages", + "Linters" + ], + "activationEvents": [ + "onLanguage:python", + "onLanguage:plaintext" + ], + "main": "../../out/client/extension", + "contributes": { + "debuggers": [ + { + "type": "python", + "label": "Python (pdb)", + "enableBreakpointsFor": { + "languageIds": [ + "python" + ] + }, + "program": "./out/client/debugger/pdb/debuggerMain.js", + "runtime": "node", + "configurationAttributes": { + "launch": { + "required": [ + "program" + ], + "properties": { + "program": { + "type": "string", + "description": "Workspace relative path to a text file.", + "default": "__init__.py" + }, + "pythonPath": { + "type": "string", + "description": "Path (fully qualified) to python executable. Use this if you want to use a custom pthon executable version.", + "default": "" + }, + "stopOnEntry": { + "type": "boolean", + "description": "Automatically stop after launch.", + "default": true + }, + "args": { + "type": "array", + "description": "List of arguments for the program", + "default": [] + } + } + } + }, + "initialConfigurations": [ + { + "name": "Python (Pdb)", + "type": "python_pdb", + "request": "launch", + "program": "__init__.py", + "stopOnEntry": true + } + ] + }, + { + "type": "python_win", + "label": "Python (Windows)", + "enableBreakpointsFor": { + "languageIds": [ + "python" + ] + }, + "program": "./out/client/debugger/vs/VSDebugger.js", + "runtime": "node", + "configurationAttributes": { + "launch": { + "required": [ + "program" + ], + "properties": { + "program": { + "type": "string", + "description": "Workspace relative path to a text file.", + "default": "__init__.py" + }, + "pythonPath": { + "type": "string", + "description": "Path (fully qualified) to python executable. Use this if you want to use a custom pthon executable version.", + "default": "" + }, + "args": { + "type": "array", + "description": "List of arguments for the program", + "default": [] + } + } + } + }, + "initialConfigurations": [ + { + "name": "Python (Windows)", + "type": "python_win", + "request": "launch", + "program": "__init__.py" + } + ] + } + ], + "configuration": { + "type": "object", + "title": "Python Configuration", "properties": { - "program": { - "type": "string", - "description": "Workspace relative path to a text file.", - "default": "__init__.py" - }, - "stopOnEntry": { - "type": "boolean", - "description": "Automatically stop after launch.", - "default": true - } + "python.maxNumberOfProblems": { + "type": "number", + "default": 100, + "description": "Controls the maximum number of problems produced by the server." + } } - } - }, - "initialConfigurations": [ - { - "name": "Python", - "type": "python", - "request": "launch", - "program": "__init__.py", - "stopOnEntry": true - } - ] - } - ], - "configuration": { - "type": "object", - "title": "Python Configuration", - "properties": { - "python.maxNumberOfProblems": { - "type": "number", - "default": 100, - "description": "Controls the maximum number of problems produced by the server." } - } + }, + "scripts": { + "vscode:prepublish": "node ./node_modules/vscode/bin/compile", + "compile": "node ./node_modules/vscode/bin/compile -watch -p ./" + }, + "dependencies": { + "long": "^3.0.3", + "named-js-regexp": "^1.3.1", + "tmp": "0.0.28", + "vscode-debugadapter": "^1.0.1", + "vscode-debugprotocol": "^1.0.1", + "vscode-languageclient": "^1.1.0" + }, + "devDependencies": { + "typescript": "^1.6.2", + "vscode": "0.10.x" } - }, - "scripts": { - "vscode:prepublish": "node ./node_modules/vscode/bin/compile", - "compile": "node ./node_modules/vscode/bin/compile -watch -p ./" - }, - "dependencies": { - "named-js-regexp": "^1.3.1", - "tmp": "0.0.28", - "vscode-debugadapter": "^1.0.1", - "vscode-debugprotocol": "^1.0.1", - "vscode-languageclient": "^1.1.0" - }, - "devDependencies": { - "typescript": "^1.6.2", - "vscode": "0.10.x" - } -} +} \ No newline at end of file diff --git a/src/client/providers/jediProxy.ts b/src/client/providers/jediProxy.ts index 52acdd68ffc2..8a37b9dbfb61 100644 --- a/src/client/providers/jediProxy.ts +++ b/src/client/providers/jediProxy.ts @@ -340,8 +340,16 @@ function createPayload(cmd: IExecutionCommand): any } function getConfig() { + //Add support for paths relative to workspace + var extraPaths = pythonSettings.autoComplete.extraPaths.map(extraPath=> { + if (path.isAbsolute(extraPath)) { + return extraPath; + } + return path.join(vscode.workspace.rootPath, extraPath); + }); + return { - extraPaths: pythonSettings.autoComplete.extraPaths, + extraPaths: extraPaths, useSnippets: false, caseInsensitiveCompletion: true, showDescriptions: true,