Source Code for Module pyspark.files

 1  # 
 2  # Licensed to the Apache Software Foundation (ASF) under one or more 
 3  # contributor license agreements.  See the NOTICE file distributed with 
 4  # this work for additional information regarding copyright ownership. 
 5  # The ASF licenses this file to You under the Apache License, Version 2.0 
 6  # (the "License"); you may not use this file except in compliance with 
 7  # the License.  You may obtain a copy of the License at 
 8  # 
 9  #    http://www.apache.org/licenses/LICENSE-2.0 
10  # 
11  # Unless required by applicable law or agreed to in writing, software 
12  # distributed under the License is distributed on an "AS IS" BASIS, 
13  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 
14  # See the License for the specific language governing permissions and 
15  # limitations under the License. 
16  # 
17   
18  import os 
19   
20   
21 -class SparkFiles(object): 
22   
23      """ 
24      Resolves paths to files added through 
25      L{SparkContext.addFile()<pyspark.context.SparkContext.addFile>}. 
26   
27      SparkFiles contains only classmethods; users should not create SparkFiles 
28      instances. 
29      """ 
30   
31      _root_directory = None 
32      _is_running_on_worker = False 
33      _sc = None 
34   
35 -    def __init__(self): 
36          raise NotImplementedError("Do not construct SparkFiles objects") 
37   
38      @classmethod 
39 -    def get(cls, filename): 
40          """ 
41          Get the absolute path of a file added through C{SparkContext.addFile()}. 
42          """ 
43          path = os.path.join(SparkFiles.getRootDirectory(), filename) 
44          return os.path.abspath(path) 
45   
46      @classmethod 
47 -    def getRootDirectory(cls): 
48          """ 
49          Get the root directory that contains files added through 
50          C{SparkContext.addFile()}. 
51          """ 
52          if cls._is_running_on_worker: 
53              return cls._root_directory 
54          else: 
55              # This will have to change if we support multiple SparkContexts: 
56              return cls._sc._jvm.org.apache.spark.SparkFiles.getRootDirectory() 
57