1
2
3 '''
4 Collector -- Crash processing client
5
6 Provide process and class level interfaces to process crash information with
7 a remote server.
8
9 @author: Christian Holler (:decoder)
10
11 @license:
12
13 This Source Code Form is subject to the terms of the Mozilla Public
14 License, v. 2.0. If a copy of the MPL was not distributed with this
15 file, You can obtain one at http://mozilla.org/MPL/2.0/.
16
17 @contact: choller@mozilla.com
18 '''
19
20
21 from __future__ import print_function
22
23 import sys
24 import os
25 import json
26 import base64
27 import argparse
28 import hashlib
29 import platform
30 import requests
31 from tempfile import mkstemp
32 from zipfile import ZipFile
33
34 BASE_DIR = os.path.dirname(os.path.realpath(__file__))
35 FTB_PATH = os.path.abspath(os.path.join(BASE_DIR, ".."))
36 sys.path += [FTB_PATH]
37
38 from FTB.ProgramConfiguration import ProgramConfiguration
39 from FTB.Running.AutoRunner import AutoRunner
40 from FTB.Signatures.CrashSignature import CrashSignature
41 from FTB.Signatures.CrashInfo import CrashInfo
42 from FTB.ConfigurationFiles import ConfigurationFiles
43
44
45 __all__ = []
46 __version__ = 0.1
47 __date__ = '2014-10-01'
48 __updated__ = '2014-10-01'
51 'Decorator to perform error checks before using remote features'
52 def decorator(self, *args, **kwargs):
53 if not self.serverHost:
54 raise RuntimeError("Must specify serverHost (configuration property: serverhost) to use remote features.")
55 if not self.serverHost:
56 raise RuntimeError("Must specify serverAuthToken (configuration property: serverauthtoken) to use remote features.")
57 if not self.tool:
58 raise RuntimeError("Must specify tool (configuration property: tool) to use remote features.")
59 return f(self, *args, **kwargs)
60 return decorator
61
63 'Decorator to perform error checks before using signature features'
64 def decorator(self, *args, **kwargs):
65 if not self.sigCacheDir:
66 raise RuntimeError("Must specify sigCacheDir (configuration property: sigdir) to use signatures.")
67 return f(self, *args, **kwargs)
68 return decorator
69
71 - def __init__(self, sigCacheDir=None, serverHost=None, serverPort=None,
72 serverProtocol=None, serverAuthToken=None,
73 clientId=None, tool=None):
74 '''
75 Initialize the Collector. This constructor will also attempt to read
76 a configuration file to populate any missing properties that have not
77 been passed to this constructor.
78
79 @type sigCacheDir: string
80 @param sigCacheDir: Directory to be used for caching signatures
81 @type serverHost: string
82 @param serverHost: Server host to contact for refreshing signatures
83 @type serverPort: int
84 @param serverPort: Server port to use when contacting server
85 @type serverAuthToken: string
86 @param serverAuthToken: Token for server authentication
87 @type clientId: string
88 @param clientId: Client ID stored in the server when submitting issues
89 @type tool: string
90 @param tool: Name of the tool that found this issue
91 '''
92 self.sigCacheDir = sigCacheDir
93 self.serverHost = serverHost
94 self.serverPort = serverPort
95 self.serverProtocol = serverProtocol
96 self.serverAuthToken = serverAuthToken
97 self.clientId = clientId
98 self.tool = tool
99
100
101
102 globalConfigFile = os.path.join(os.path.expanduser("~"), ".fuzzmanagerconf")
103 if os.path.exists(globalConfigFile):
104 configInstance = ConfigurationFiles([ globalConfigFile ])
105 globalConfig = configInstance.mainConfig
106
107 if self.sigCacheDir == None and "sigdir" in globalConfig:
108 self.sigCacheDir = globalConfig["sigdir"]
109
110 if self.serverHost == None and "serverhost" in globalConfig:
111 self.serverHost = globalConfig["serverhost"]
112
113 if self.serverPort == None and "serverport" in globalConfig:
114 self.serverPort = globalConfig["serverport"]
115
116 if self.serverProtocol == None and "serverproto" in globalConfig:
117 self.serverProtocol = globalConfig["serverproto"]
118
119 if self.serverAuthToken == None:
120 if "serverauthtoken" in globalConfig:
121 self.serverAuthToken = globalConfig["serverauthtoken"]
122 elif "serverauthtokenfile" in globalConfig:
123 with open(globalConfig["serverauthtokenfile"]) as f:
124 self.serverAuthToken = f.read().rstrip()
125
126 if self.clientId == None and "clientid" in globalConfig:
127 self.clientId = globalConfig["clientid"]
128
129 if self.tool == None and "tool" in globalConfig:
130 self.tool = globalConfig["tool"]
131
132
133
134 if self.serverProtocol == None:
135 self.serverProtocol = "https"
136
137
138 if self.serverPort == None:
139 if self.serverProtocol == "https":
140 self.serverPort = 433
141 else:
142 self.serverPort = 80
143
144 if self.serverHost != None and self.clientId == None:
145 self.clientId = platform.node()
146
147 @remote_checks
148 @signature_checks
150 '''
151 Refresh signatures by contacting the server, downloading new signatures
152 and invalidating old ones.
153 '''
154 url = "%s://%s:%s/crashmanager/files/signatures.zip" % (self.serverProtocol, self.serverHost, self.serverPort)
155
156
157 response = requests.get(url, stream=True, auth=('fuzzmanager', self.serverAuthToken))
158
159 if response.status_code != requests.codes["ok"]:
160 raise self.__serverError(response)
161
162 (zipFileFd, zipFileName) = mkstemp(prefix="fuzzmanager-signatures")
163
164 with os.fdopen(zipFileFd, 'w') as zipFile:
165 for chunk in response.iter_content(chunk_size=1024):
166 if chunk:
167 zipFile.write(chunk)
168 zipFile.flush()
169
170 self.refreshFromZip(zipFileName)
171 os.remove(zipFileName)
172
173 @signature_checks
175 '''
176 Refresh signatures from a local zip file, adding new signatures
177 and invalidating old ones. (This is a non-standard use case;
178 you probably want to use refresh() instead.)
179 '''
180 with ZipFile(zipFileName, "r") as zipFile:
181 if zipFile.testzip() != None:
182 raise RuntimeError("Bad CRC for downloaded zipfile %s" % zipFileName)
183
184
185 for sigFile in os.listdir(self.sigCacheDir):
186 if sigFile.endswith(".signature") or sigFile.endswith(".metadata"):
187 os.remove(os.path.join(self.sigCacheDir, sigFile))
188 else:
189 print("Warning: Skipping deletion of non-signature file: %s" % sigFile, file=sys.stderr)
190
191 zipFile.extractall(self.sigCacheDir)
192
193 @remote_checks
194 - def submit(self, crashInfo, testCase=None, testCaseQuality=0, metaData=None):
195 '''
196 Submit the given crash information and an optional testcase/metadata
197 to the server for processing and storage.
198
199 @type crashInfo: CrashInfo
200 @param crashInfo: CrashInfo instance obtained from L{CrashInfo.fromRawCrashData}
201
202 @type testCase: string
203 @param testCase: A file containing a testcase for reproduction
204
205 @type testCaseQuality: int
206 @param testCaseQuality: A value indicating the quality of the test (less is better)
207
208 @type metaData: map
209 @param metaData: A map containing arbitrary (application-specific) data which
210 will be stored on the server in JSON format. This metadata is combined
211 with possible metadata stored in the L{ProgramConfiguration} inside crashInfo.
212 '''
213 url = "%s://%s:%s/crashmanager/rest/crashes/" % (self.serverProtocol, self.serverHost, self.serverPort)
214
215
216 data = {}
217
218 data["rawStdout"] = os.linesep.join(crashInfo.rawStdout)
219 data["rawStderr"] = os.linesep.join(crashInfo.rawStderr)
220 data["rawCrashData"] = os.linesep.join(crashInfo.rawCrashData)
221
222 if testCase:
223 (testCaseData, isBinary) = Collector.read_testcase(testCase)
224
225 if isBinary:
226 testCaseData = base64.b64encode(testCaseData)
227
228 data["testcase"] = testCaseData
229 data["testcase_isbinary"] = isBinary
230 data["testcase_quality"] = testCaseQuality
231 data["testcase_ext"] = os.path.splitext(testCase)[1][1:]
232
233 data["platform"] = crashInfo.configuration.platform
234 data["product"] = crashInfo.configuration.product
235 data["os"] = crashInfo.configuration.os
236
237 if crashInfo.configuration.version:
238 data["product_version"] = crashInfo.configuration.version
239
240 data["client"] = self.clientId
241 data["tool"] = self.tool
242
243 if crashInfo.configuration.metadata or metaData:
244 aggrMetaData = {}
245
246 if crashInfo.configuration.metadata:
247 aggrMetaData.update(crashInfo.configuration.metadata)
248
249 if metaData:
250 aggrMetaData.update(metaData)
251
252 data["metadata"] = json.dumps(aggrMetaData)
253
254 if crashInfo.configuration.env:
255 data["env"] = json.dumps(crashInfo.configuration.env)
256
257 if crashInfo.configuration.args:
258 data["args"] = json.dumps(crashInfo.configuration.args)
259
260 response = requests.post(url, data, headers=dict(Authorization="Token %s" % self.serverAuthToken))
261
262 if response.status_code != requests.codes["created"]:
263 raise self.__serverError(response)
264
265 @signature_checks
267 '''
268 Searches within the local signature cache directory for a signature matching the
269 given crash.
270
271 @type crashInfo: CrashInfo
272 @param crashInfo: CrashInfo instance obtained from L{CrashInfo.fromRawCrashData}
273
274 @rtype: tuple
275 @return: Tuple containing filename of the signature and metadata matching, or None if no match.
276 '''
277
278 cachedSigFiles = os.listdir(self.sigCacheDir)
279
280 for sigFile in cachedSigFiles:
281 if not sigFile.endswith('.signature'):
282 continue
283
284 sigFile = os.path.join(self.sigCacheDir, sigFile)
285 if not os.path.isdir(sigFile):
286 with open(sigFile) as f:
287 sigData = f.read()
288 crashSig = CrashSignature(sigData)
289 if crashSig.matches(crashInfo):
290 metadataFile = sigFile.replace('.signature', '.metadata')
291 metadata = None
292 if os.path.exists(metadataFile):
293 with open(metadataFile) as m:
294 metadata = json.loads(m.read())
295
296 return (sigFile, metadata)
297
298 return (None, None)
299
300 @signature_checks
301 - def generate(self, crashInfo, forceCrashAddress=None, forceCrashInstruction=None, numFrames=None):
302 '''
303 Generates a signature in the local cache directory. It will be deleted when L{refresh} is called
304 on the same local cache directory.
305
306 @type crashInfo: CrashInfo
307 @param crashInfo: CrashInfo instance obtained from L{CrashInfo.fromRawCrashData}
308
309 @type forceCrashAddress: bool
310 @param forceCrashAddress: Force including the crash address into the signature
311 @type forceCrashInstruction: bool
312 @param forceCrashInstruction: Force including the crash instruction into the signature (GDB only)
313 @type numFrames: int
314 @param numFrames: How many frames to include in the signature
315
316 @rtype: string
317 @return: File containing crash signature in JSON format
318 '''
319
320 sig = crashInfo.createCrashSignature(forceCrashAddress, forceCrashInstruction, numFrames)
321
322 if not sig:
323 return None
324
325
326 return self.__store_signature_hashed(sig)
327
328 @remote_checks
330 '''
331 Download the testcase for the specified crashId.
332
333 @type crashId: int
334 @param crashId: ID of the requested crash entry on the server side
335
336 @rtype: tuple
337 @return: Tuple containing name of the file where the test was stored and the raw JSON response
338 '''
339 if not self.serverHost:
340 raise RuntimeError("Must specify serverHost to use remote features.")
341
342 url = "%s://%s:%s/crashmanager/rest/crashes/%s/" % (self.serverProtocol, self.serverHost, self.serverPort, crashId)
343
344 response = requests.get(url, headers=dict(Authorization="Token %s" % self.serverAuthToken))
345
346 if response.status_code != requests.codes["ok"]:
347 raise self.__serverError(response)
348
349 json = response.json()
350
351 if not isinstance(json, dict):
352 raise RuntimeError("Server sent malformed JSON response: %s" % json)
353
354 if not json["testcase"]:
355 return None
356
357 url = "%s://%s:%s/crashmanager/%s" % (self.serverProtocol, self.serverHost, self.serverPort, json["testcase"])
358 response = requests.get(url, auth=('fuzzmanager', self.serverAuthToken))
359
360 if response.status_code != requests.codes["ok"]:
361 raise self.__serverError(response)
362
363 localFile = os.path.basename(json["testcase"])
364 with open(localFile, 'w') as f:
365 f.write(response.content)
366
367 return (localFile, json)
368
370 '''
371 Store a signature, using the sha1 hash hex representation as filename.
372
373 @type signature: CrashSignature
374 @param signature: CrashSignature to store
375
376 @rtype: string
377 @return: Name of the file that the signature was written to
378
379 '''
380 h = hashlib.new('sha1')
381 h.update(str(signature))
382 sigfile = os.path.join(self.sigCacheDir, h.hexdigest() + ".signature")
383 with open(sigfile, 'w') as f:
384 f.write(str(signature))
385
386 return sigfile
387
388 @staticmethod
390 return RuntimeError("Server unexpectedly responded with status code %s: %s" %
391 (response.status_code, response.text))
392
393 @staticmethod
395 '''
396 Read a testcase file, return the content and indicate if it is binary or not.
397
398 @type testCase: string
399 @param testCase: Filename of the file to open
400
401 @rtype: tuple(string, bool)
402 @return: Tuple containing the file contents and a boolean indicating if the content is binary
403
404 '''
405 with open(testCase) as f:
406 testCaseData = f.read()
407
408 textBytes = bytearray([7,8,9,10,12,13,27]) + bytearray(range(0x20, 0x100))
409 isBinary = lambda input: bool(input.translate(None, textBytes))
410
411 return (testCaseData, isBinary(testCaseData))
412
413 -def main(argv=None):
414 '''Command line options.'''
415
416 program_name = os.path.basename(sys.argv[0])
417 program_version = "v%s" % __version__
418 program_build_date = "%s" % __updated__
419
420 program_version_string = '%%prog %s (%s)' % (program_version, program_build_date)
421
422 if argv is None:
423 argv = sys.argv[1:]
424
425
426 parser = argparse.ArgumentParser()
427
428 parser.add_argument('--version', action='version', version=program_version_string)
429
430
431 parser.add_argument("--stdout", dest="stdout", help="File containing STDOUT data", metavar="FILE")
432 parser.add_argument("--stderr", dest="stderr", help="File containing STDERR data", metavar="FILE")
433 parser.add_argument("--crashdata", dest="crashdata", help="File containing external crash data", metavar="FILE")
434
435
436 parser.add_argument("--refresh", dest="refresh", action='store_true', help="Perform a signature refresh")
437 parser.add_argument("--submit", dest="submit", action='store_true', help="Submit a signature to the server")
438 parser.add_argument("--search", dest="search", action='store_true', help="Search cached signatures for the given crash")
439 parser.add_argument("--generate", dest="generate", action='store_true', help="Create a (temporary) local signature in the cache directory")
440 parser.add_argument("--autosubmit", dest="autosubmit", action='store_true', help="Go into auto-submit mode. In this mode, all remaining arguments are interpreted as the crashing command. This tool will automatically obtain GDB crash information and submit it.")
441 parser.add_argument("--download", dest="download", type=int, help="Download the testcase for the specified crash entry", metavar="ID")
442
443
444 parser.add_argument("--sigdir", dest="sigdir", help="Signature cache directory", metavar="DIR")
445 parser.add_argument("--serverhost", dest="serverhost", help="Server hostname for remote signature management", metavar="HOST")
446 parser.add_argument("--serverport", dest="serverport", type=int, help="Server port to use", metavar="PORT")
447 parser.add_argument("--serverproto", dest="serverproto", help="Server protocol to use (default is https)", metavar="PROTO")
448 parser.add_argument("--serverauthtokenfile", dest="serverauthtokenfile", help="File containing the server authentication token", metavar="FILE")
449 parser.add_argument("--clientid", dest="clientid", help="Client ID to use when submitting issues", metavar="ID")
450 parser.add_argument("--platform", dest="platform", help="Platform this crash appeared on", metavar="(x86|x86-64|arm)")
451 parser.add_argument("--product", dest="product", help="Product this crash appeared on", metavar="PRODUCT")
452 parser.add_argument("--productversion", dest="product_version", help="Product version this crash appeared on", metavar="VERSION")
453 parser.add_argument("--os", dest="os", help="OS this crash appeared on", metavar="(windows|linux|macosx|b2g|android)")
454 parser.add_argument("--tool", dest="tool", help="Name of the tool that found this issue", metavar="NAME")
455 parser.add_argument('--args', dest='args', nargs='+', type=str, help="List of program arguments. Backslashes can be used for escaping and are stripped.")
456 parser.add_argument('--env', dest='env', nargs='+', type=str, help="List of environment variables in the form 'KEY=VALUE'")
457 parser.add_argument('--metadata', dest='metadata', nargs='+', type=str, help="List of metadata variables in the form 'KEY=VALUE'")
458 parser.add_argument("--binary", dest="binary", help="Binary that has a configuration file for reading", metavar="BINARY")
459
460
461 parser.add_argument("--testcase", dest="testcase", help="File containing testcase", metavar="FILE")
462 parser.add_argument("--testcasequality", dest="testcasequality", default="0", help="Integer indicating test case quality (0 is best and default)", metavar="VAL")
463
464
465 parser.add_argument("--forcecrashaddr", dest="forcecrashaddr", action='store_true', help="Force including the crash address into the signature")
466 parser.add_argument("--forcecrashinst", dest="forcecrashinst", action='store_true', help="Force including the crash instruction into the signature (GDB only)")
467 parser.add_argument("--numframes", dest="numframes", default=8, type=int, help="How many frames to include into the signature (default is 8)")
468
469 parser.add_argument('rargs', nargs=argparse.REMAINDER)
470
471 if len(argv) == 0:
472 parser.print_help()
473 return 2
474
475
476 opts = parser.parse_args(argv)
477
478
479 actions = [ "refresh", "submit", "search", "generate", "autosubmit", "download" ]
480
481 haveAction = False
482 for action in actions:
483 if getattr(opts, action):
484 if haveAction:
485 print("Error: Cannot specify multiple actions at the same time", file=sys.stderr)
486 return 2
487 haveAction = True
488 if not haveAction:
489 print("Error: Must specify an action", file=sys.stderr)
490 return 2
491
492
493
494 if opts.autosubmit:
495 if not opts.rargs:
496 print("Error: Action --autosubmit requires test arguments to be specified", file=sys.stderr)
497 return 2
498
499
500 if not opts.binary:
501 opts.binary = opts.rargs[0]
502
503
504
505 testcase = opts.testcase
506 testcaseidx = None
507 if testcase == None:
508 for idx, arg in enumerate(opts.rargs[1:]):
509 if os.path.exists(arg):
510 if testcase:
511 print("Error: Multiple potential testcases specified on command line. Must explicitely specify test using --testcase.")
512 return 2
513 testcase = arg
514 testcaseidx = idx
515
516
517
518 if opts.binary and not os.path.exists(opts.binary):
519 print("Error: Specified binary does not exist: %s" % opts.binary)
520 return 2
521
522 stdout = None
523 stderr = None
524 crashdata = None
525 crashInfo = None
526 args = None
527 env = None
528 metadata = {}
529
530 if opts.search or opts.generate or opts.submit or opts.autosubmit:
531 if opts.metadata:
532 metadata.update(dict(kv.split('=', 1) for kv in opts.metadata))
533
534 if opts.autosubmit:
535
536
537
538 if testcaseidx == len(opts.rargs[1:]) - 1:
539 args = opts.rargs[1:-1]
540 else:
541 args = opts.rargs[1:]
542 if testcaseidx != None:
543 args[testcaseidx] = "TESTFILE"
544 else:
545 if opts.args:
546 args = [arg.replace('\\', '') for arg in opts.args]
547
548 if opts.env:
549 env = dict(kv.split('=', 1) for kv in opts.env)
550
551
552 configuration = None
553
554
555 if opts.binary:
556 configuration = ProgramConfiguration.fromBinary(opts.binary)
557 if configuration:
558 if env:
559 configuration.addEnvironmentVariables(env)
560 if args:
561 configuration.addProgramArguments(args)
562 if metadata:
563 configuration.addMetadata(metadata)
564
565
566 if configuration == None:
567 if opts.platform == None or opts.product == None or opts.os == None:
568 print("Error: Must specify/configure at least --platform, --product and --os", file=sys.stderr)
569 return 2
570
571 configuration = ProgramConfiguration(opts.product, opts.platform, opts.os, opts.product_version, env, args, metadata)
572
573
574 if not opts.autosubmit:
575 if opts.stderr == None and opts.crashdata == None:
576 print("Error: Must specify at least either --stderr or --crashdata file", file=sys.stderr)
577 return 2
578
579 if opts.stdout:
580 with open(opts.stdout) as f:
581 stdout = f.read()
582
583 if opts.stderr:
584 with open(opts.stderr) as f:
585 stderr = f.read()
586
587 if opts.crashdata:
588 with open(opts.crashdata) as f:
589 crashdata = f.read()
590
591 crashInfo = CrashInfo.fromRawCrashData(stdout, stderr, configuration, auxCrashData=crashdata)
592 if opts.testcase:
593 (testCaseData, isBinary) = Collector.read_testcase(opts.testcase)
594 if not isBinary:
595 crashInfo.testcase = testCaseData
596
597 serverauthtoken = None
598 if opts.serverauthtokenfile:
599 with open(opts.serverauthtokenfile) as f:
600 serverauthtoken = f.read().rstrip()
601
602 collector = Collector(opts.sigdir, opts.serverhost, opts.serverport, opts.serverproto, serverauthtoken, opts.clientid, opts.tool)
603
604 if opts.refresh:
605 collector.refresh()
606 return 0
607
608 if opts.submit:
609 testcase = opts.testcase
610 collector.submit(crashInfo, testcase, opts.testcasequality, metadata)
611 return 0
612
613 if opts.search:
614 (sig, metadata) = collector.search(crashInfo)
615 if sig == None:
616 print("No match found")
617 return 3
618 print(sig)
619 if metadata:
620 print(json.dumps(metadata, indent=4))
621 return 0
622
623 if opts.generate:
624 sigFile = collector.generate(crashInfo, opts.forcecrashaddr, opts.forcecrashinst, opts.numframes)
625 if not sigFile:
626 print("Failed to generate a signature for the given crash information.", file=sys.stderr)
627 return 2
628 print(sigFile)
629 return 0
630
631 if opts.autosubmit:
632 runner = AutoRunner.fromBinaryArgs(opts.rargs[0], opts.rargs[1:])
633 if runner.run():
634 crashInfo = runner.getCrashInfo(configuration)
635 collector.submit(crashInfo, testcase, opts.testcasequality, metadata)
636 else:
637 print("Error: Failed to reproduce the given crash, cannot submit.", file=sys.stderr)
638 return 2
639
640 if opts.download:
641 (retFile, retJSON) = collector.download(opts.download)
642 if not retFile:
643 print("Specified crash entry does not have a testcase", file=sys.stderr)
644 return 2
645
646 if "args" in retJSON and retJSON["args"]:
647 args = json.loads(retJSON["args"])
648 print("Command line arguments: %s" % " ".join(args))
649 print("")
650
651 if "env" in retJSON and retJSON["env"]:
652 env = json.loads(retJSON["env"])
653 print("Environment variables: %s", " ".join([ "%s = %s" % (k,v) for (k,v) in env.items()]))
654 print("")
655
656 if "metadata" in retJSON and retJSON["metadata"]:
657 metadata = json.loads(retJSON["metadata"])
658 print("== Metadata ==")
659 for k, v in metadata.items():
660 print("%s = %s" % (k,v))
661 print("")
662
663
664 print(retFile)
665 return 0
666
667 if __name__ == "__main__":
668 sys.exit(main())
669