secondPysparkHelpFile.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark import SparkConf,SparkContext\n",
    "\n",
    "sc = SparkContext(conf=SparkConf().setAppName(\"myApp\").setMaster(\"local\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Writing trial.txt\n"
     ]
    }
   ],
   "source": [
    "%%writefile trial.txt\n",
    "hello hello how are you\n",
    "what is yyour name\n",
    "My name is pyspark\n",
    "and I am good at data science and that too faster than hadoop"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "textfileRDD = sc.textFile(\"trial.txt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['__add__',\n",
       " '__class__',\n",
       " '__delattr__',\n",
       " '__dict__',\n",
       " '__dir__',\n",
       " '__doc__',\n",
       " '__eq__',\n",
       " '__format__',\n",
       " '__ge__',\n",
       " '__getattribute__',\n",
       " '__getnewargs__',\n",
       " '__gt__',\n",
       " '__hash__',\n",
       " '__init__',\n",
       " '__init_subclass__',\n",
       " '__le__',\n",
       " '__lt__',\n",
       " '__module__',\n",
       " '__ne__',\n",
       " '__new__',\n",
       " '__reduce__',\n",
       " '__reduce_ex__',\n",
       " '__repr__',\n",
       " '__setattr__',\n",
       " '__sizeof__',\n",
       " '__str__',\n",
       " '__subclasshook__',\n",
       " '__weakref__',\n",
       " '_computeFractionForSampleSize',\n",
       " '_defaultReducePartitions',\n",
       " '_id',\n",
       " '_jrdd',\n",
       " '_jrdd_deserializer',\n",
       " '_memory_limit',\n",
       " '_pickled',\n",
       " '_reserialize',\n",
       " '_to_java_object_rdd',\n",
       " 'aggregate',\n",
       " 'aggregateByKey',\n",
       " 'cache',\n",
       " 'cartesian',\n",
       " 'checkpoint',\n",
       " 'coalesce',\n",
       " 'cogroup',\n",
       " 'collect',\n",
       " 'collectAsMap',\n",
       " 'combineByKey',\n",
       " 'context',\n",
       " 'count',\n",
       " 'countApprox',\n",
       " 'countApproxDistinct',\n",
       " 'countByKey',\n",
       " 'countByValue',\n",
       " 'ctx',\n",
       " 'distinct',\n",
       " 'filter',\n",
       " 'first',\n",
       " 'flatMap',\n",
       " 'flatMapValues',\n",
       " 'fold',\n",
       " 'foldByKey',\n",
       " 'foreach',\n",
       " 'foreachPartition',\n",
       " 'fullOuterJoin',\n",
       " 'getCheckpointFile',\n",
       " 'getNumPartitions',\n",
       " 'getStorageLevel',\n",
       " 'glom',\n",
       " 'groupBy',\n",
       " 'groupByKey',\n",
       " 'groupWith',\n",
       " 'histogram',\n",
       " 'id',\n",
       " 'intersection',\n",
       " 'isCheckpointed',\n",
       " 'isEmpty',\n",
       " 'isLocallyCheckpointed',\n",
       " 'is_cached',\n",
       " 'is_checkpointed',\n",
       " 'join',\n",
       " 'keyBy',\n",
       " 'keys',\n",
       " 'leftOuterJoin',\n",
       " 'localCheckpoint',\n",
       " 'lookup',\n",
       " 'map',\n",
       " 'mapPartitions',\n",
       " 'mapPartitionsWithIndex',\n",
       " 'mapPartitionsWithSplit',\n",
       " 'mapValues',\n",
       " 'max',\n",
       " 'mean',\n",
       " 'meanApprox',\n",
       " 'min',\n",
       " 'name',\n",
       " 'partitionBy',\n",
       " 'partitioner',\n",
       " 'persist',\n",
       " 'pipe',\n",
       " 'randomSplit',\n",
       " 'reduce',\n",
       " 'reduceByKey',\n",
       " 'reduceByKeyLocally',\n",
       " 'repartition',\n",
       " 'repartitionAndSortWithinPartitions',\n",
       " 'rightOuterJoin',\n",
       " 'sample',\n",
       " 'sampleByKey',\n",
       " 'sampleStdev',\n",
       " 'sampleVariance',\n",
       " 'saveAsHadoopDataset',\n",
       " 'saveAsHadoopFile',\n",
       " 'saveAsNewAPIHadoopDataset',\n",
       " 'saveAsNewAPIHadoopFile',\n",
       " 'saveAsPickleFile',\n",
       " 'saveAsSequenceFile',\n",
       " 'saveAsTextFile',\n",
       " 'setName',\n",
       " 'sortBy',\n",
       " 'sortByKey',\n",
       " 'stats',\n",
       " 'stdev',\n",
       " 'subtract',\n",
       " 'subtractByKey',\n",
       " 'sum',\n",
       " 'sumApprox',\n",
       " 'take',\n",
       " 'takeOrdered',\n",
       " 'takeSample',\n",
       " 'toDebugString',\n",
       " 'toLocalIterator',\n",
       " 'top',\n",
       " 'treeAggregate',\n",
       " 'treeReduce',\n",
       " 'union',\n",
       " 'unpersist',\n",
       " 'values',\n",
       " 'variance',\n",
       " 'zip',\n",
       " 'zipWithIndex',\n",
       " 'zipWithUniqueId']"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dir(textfileRDD)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Help on RDD in module pyspark.rdd object:\n",
      "\n",
      "class RDD(builtins.object)\n",
      " |  A Resilient Distributed Dataset (RDD), the basic abstraction in Spark.\n",
      " |  Represents an immutable, partitioned collection of elements that can be\n",
      " |  operated on in parallel.\n",
      " |  \n",
      " |  Methods defined here:\n",
      " |  \n",
      " |  __add__(self, other)\n",
      " |      Return the union of this RDD and another one.\n",
      " |      \n",
      " |      >>> rdd = sc.parallelize([1, 1, 2, 3])\n",
      " |      >>> (rdd + rdd).collect()\n",
      " |      [1, 1, 2, 3, 1, 1, 2, 3]\n",
      " |  \n",
      " |  __getnewargs__(self)\n",
      " |  \n",
      " |  __init__(self, jrdd, ctx, jrdd_deserializer=AutoBatchedSerializer(PickleSerializer()))\n",
      " |      Initialize self.  See help(type(self)) for accurate signature.\n",
      " |  \n",
      " |  __repr__(self)\n",
      " |      Return repr(self).\n",
      " |  \n",
      " |  aggregate(self, zeroValue, seqOp, combOp)\n",
      " |      Aggregate the elements of each partition, and then the results for all\n",
      " |      the partitions, using a given combine functions and a neutral \"zero\n",
      " |      value.\"\n",
      " |      \n",
      " |      The functions C{op(t1, t2)} is allowed to modify C{t1} and return it\n",
      " |      as its result value to avoid object allocation; however, it should not\n",
      " |      modify C{t2}.\n",
      " |      \n",
      " |      The first function (seqOp) can return a different result type, U, than\n",
      " |      the type of this RDD. Thus, we need one operation for merging a T into\n",
      " |      an U and one operation for merging two U\n",
      " |      \n",
      " |      >>> seqOp = (lambda x, y: (x[0] + y, x[1] + 1))\n",
      " |      >>> combOp = (lambda x, y: (x[0] + y[0], x[1] + y[1]))\n",
      " |      >>> sc.parallelize([1, 2, 3, 4]).aggregate((0, 0), seqOp, combOp)\n",
      " |      (10, 4)\n",
      " |      >>> sc.parallelize([]).aggregate((0, 0), seqOp, combOp)\n",
      " |      (0, 0)\n",
      " |  \n",
      " |  aggregateByKey(self, zeroValue, seqFunc, combFunc, numPartitions=None, partitionFunc=<function portable_hash at 0x7f4817231a60>)\n",
      " |      Aggregate the values of each key, using given combine functions and a neutral\n",
      " |      \"zero value\". This function can return a different result type, U, than the type\n",
      " |      of the values in this RDD, V. Thus, we need one operation for merging a V into\n",
      " |      a U and one operation for merging two U's, The former operation is used for merging\n",
      " |      values within a partition, and the latter is used for merging values between\n",
      " |      partitions. To avoid memory allocation, both of these functions are\n",
      " |      allowed to modify and return their first argument instead of creating a new U.\n",
      " |  \n",
      " |  cache(self)\n",
      " |      Persist this RDD with the default storage level (C{MEMORY_ONLY}).\n",
      " |  \n",
      " |  cartesian(self, other)\n",
      " |      Return the Cartesian product of this RDD and another one, that is, the\n",
      " |      RDD of all pairs of elements C{(a, b)} where C{a} is in C{self} and\n",
      " |      C{b} is in C{other}.\n",
      " |      \n",
      " |      >>> rdd = sc.parallelize([1, 2])\n",
      " |      >>> sorted(rdd.cartesian(rdd).collect())\n",
      " |      [(1, 1), (1, 2), (2, 1), (2, 2)]\n",
      " |  \n",
      " |  checkpoint(self)\n",
      " |      Mark this RDD for checkpointing. It will be saved to a file inside the\n",
      " |      checkpoint directory set with L{SparkContext.setCheckpointDir()} and\n",
      " |      all references to its parent RDDs will be removed. This function must\n",
      " |      be called before any job has been executed on this RDD. It is strongly\n",
      " |      recommended that this RDD is persisted in memory, otherwise saving it\n",
      " |      on a file will require recomputation.\n",
      " |  \n",
      " |  coalesce(self, numPartitions, shuffle=False)\n",
      " |      Return a new RDD that is reduced into `numPartitions` partitions.\n",
      " |      \n",
      " |      >>> sc.parallelize([1, 2, 3, 4, 5], 3).glom().collect()\n",
      " |      [[1], [2, 3], [4, 5]]\n",
      " |      >>> sc.parallelize([1, 2, 3, 4, 5], 3).coalesce(1).glom().collect()\n",
      " |      [[1, 2, 3, 4, 5]]\n",
      " |  \n",
      " |  cogroup(self, other, numPartitions=None)\n",
      " |      For each key k in C{self} or C{other}, return a resulting RDD that\n",
      " |      contains a tuple with the list of values for that key in C{self} as\n",
      " |      well as C{other}.\n",
      " |      \n",
      " |      >>> x = sc.parallelize([(\"a\", 1), (\"b\", 4)])\n",
      " |      >>> y = sc.parallelize([(\"a\", 2)])\n",
      " |      >>> [(x, tuple(map(list, y))) for x, y in sorted(list(x.cogroup(y).collect()))]\n",
      " |      [('a', ([1], [2])), ('b', ([4], []))]\n",
      " |  \n",
      " |  collect(self)\n",
      " |      Return a list that contains all of the elements in this RDD.\n",
      " |      \n",
      " |      .. note:: This method should only be used if the resulting array is expected\n",
      " |          to be small, as all the data is loaded into the driver's memory.\n",
      " |  \n",
      " |  collectAsMap(self)\n",
      " |      Return the key-value pairs in this RDD to the master as a dictionary.\n",
      " |      \n",
      " |      .. note:: this method should only be used if the resulting data is expected\n",
      " |          to be small, as all the data is loaded into the driver's memory.\n",
      " |      \n",
      " |      >>> m = sc.parallelize([(1, 2), (3, 4)]).collectAsMap()\n",
      " |      >>> m[1]\n",
      " |      2\n",
      " |      >>> m[3]\n",
      " |      4\n",
      " |  \n",
      " |  combineByKey(self, createCombiner, mergeValue, mergeCombiners, numPartitions=None, partitionFunc=<function portable_hash at 0x7f4817231a60>)\n",
      " |      Generic function to combine the elements for each key using a custom\n",
      " |      set of aggregation functions.\n",
      " |      \n",
      " |      Turns an RDD[(K, V)] into a result of type RDD[(K, C)], for a \"combined\n",
      " |      type\" C.\n",
      " |      \n",
      " |      Users provide three functions:\n",
      " |      \n",
      " |          - C{createCombiner}, which turns a V into a C (e.g., creates\n",
      " |            a one-element list)\n",
      " |          - C{mergeValue}, to merge a V into a C (e.g., adds it to the end of\n",
      " |            a list)\n",
      " |          - C{mergeCombiners}, to combine two C's into a single one.\n",
      " |      \n",
      " |      In addition, users can control the partitioning of the output RDD.\n",
      " |      \n",
      " |      .. note:: V and C can be different -- for example, one might group an RDD of type\n",
      " |          (Int, Int) into an RDD of type (Int, List[Int]).\n",
      " |      \n",
      " |      >>> x = sc.parallelize([(\"a\", 1), (\"b\", 1), (\"a\", 1)])\n",
      " |      >>> def add(a, b): return a + str(b)\n",
      " |      >>> sorted(x.combineByKey(str, add, add).collect())\n",
      " |      [('a', '11'), ('b', '1')]\n",
      " |  \n",
      " |  count(self)\n",
      " |      Return the number of elements in this RDD.\n",
      " |      \n",
      " |      >>> sc.parallelize([2, 3, 4]).count()\n",
      " |      3\n",
      " |  \n",
      " |  countApprox(self, timeout, confidence=0.95)\n",
      " |      .. note:: Experimental\n",
      " |      \n",
      " |      Approximate version of count() that returns a potentially incomplete\n",
      " |      result within a timeout, even if not all tasks have finished.\n",
      " |      \n",
      " |      >>> rdd = sc.parallelize(range(1000), 10)\n",
      " |      >>> rdd.countApprox(1000, 1.0)\n",
      " |      1000\n",
      " |  \n",
      " |  countApproxDistinct(self, relativeSD=0.05)\n",
      " |      .. note:: Experimental\n",
      " |      \n",
      " |      Return approximate number of distinct elements in the RDD.\n",
      " |      \n",
      " |      The algorithm used is based on streamlib's implementation of\n",
      " |      `\"HyperLogLog in Practice: Algorithmic Engineering of a State\n",
      " |      of The Art Cardinality Estimation Algorithm\", available here\n",
      " |      <http://dx.doi.org/10.1145/2452376.2452456>`_.\n",
      " |      \n",
      " |      :param relativeSD: Relative accuracy. Smaller values create\n",
      " |                         counters that require more space.\n",
      " |                         It must be greater than 0.000017.\n",
      " |      \n",
      " |      >>> n = sc.parallelize(range(1000)).map(str).countApproxDistinct()\n",
      " |      >>> 900 < n < 1100\n",
      " |      True\n",
      " |      >>> n = sc.parallelize([i % 20 for i in range(1000)]).countApproxDistinct()\n",
      " |      >>> 16 < n < 24\n",
      " |      True\n",
      " |  \n",
      " |  countByKey(self)\n",
      " |      Count the number of elements for each key, and return the result to the\n",
      " |      master as a dictionary.\n",
      " |      \n",
      " |      >>> rdd = sc.parallelize([(\"a\", 1), (\"b\", 1), (\"a\", 1)])\n",
      " |      >>> sorted(rdd.countByKey().items())\n",
      " |      [('a', 2), ('b', 1)]\n",
      " |  \n",
      " |  countByValue(self)\n",
      " |      Return the count of each unique value in this RDD as a dictionary of\n",
      " |      (value, count) pairs.\n",
      " |      \n",
      " |      >>> sorted(sc.parallelize([1, 2, 1, 2, 2], 2).countByValue().items())\n",
      " |      [(1, 2), (2, 3)]\n",
      " |  \n",
      " |  distinct(self, numPartitions=None)\n",
      " |      Return a new RDD containing the distinct elements in this RDD.\n",
      " |      \n",
      " |      >>> sorted(sc.parallelize([1, 1, 2, 3]).distinct().collect())\n",
      " |      [1, 2, 3]\n",
      " |  \n",
      " |  filter(self, f)\n",
      " |      Return a new RDD containing only the elements that satisfy a predicate.\n",
      " |      \n",
      " |      >>> rdd = sc.parallelize([1, 2, 3, 4, 5])\n",
      " |      >>> rdd.filter(lambda x: x % 2 == 0).collect()\n",
      " |      [2, 4]\n",
      " |  \n",
      " |  first(self)\n",
      " |      Return the first element in this RDD.\n",
      " |      \n",
      " |      >>> sc.parallelize([2, 3, 4]).first()\n",
      " |      2\n",
      " |      >>> sc.parallelize([]).first()\n",
      " |      Traceback (most recent call last):\n",
      " |          ...\n",
      " |      ValueError: RDD is empty\n",
      " |  \n",
      " |  flatMap(self, f, preservesPartitioning=False)\n",
      " |      Return a new RDD by first applying a function to all elements of this\n",
      " |      RDD, and then flattening the results.\n",
      " |      \n",
      " |      >>> rdd = sc.parallelize([2, 3, 4])\n",
      " |      >>> sorted(rdd.flatMap(lambda x: range(1, x)).collect())\n",
      " |      [1, 1, 1, 2, 2, 3]\n",
      " |      >>> sorted(rdd.flatMap(lambda x: [(x, x), (x, x)]).collect())\n",
      " |      [(2, 2), (2, 2), (3, 3), (3, 3), (4, 4), (4, 4)]\n",
      " |  \n",
      " |  flatMapValues(self, f)\n",
      " |      Pass each value in the key-value pair RDD through a flatMap function\n",
      " |      without changing the keys; this also retains the original RDD's\n",
      " |      partitioning.\n",
      " |      \n",
      " |      >>> x = sc.parallelize([(\"a\", [\"x\", \"y\", \"z\"]), (\"b\", [\"p\", \"r\"])])\n",
      " |      >>> def f(x): return x\n",
      " |      >>> x.flatMapValues(f).collect()\n",
      " |      [('a', 'x'), ('a', 'y'), ('a', 'z'), ('b', 'p'), ('b', 'r')]\n",
      " |  \n",
      " |  fold(self, zeroValue, op)\n",
      " |      Aggregate the elements of each partition, and then the results for all\n",
      " |      the partitions, using a given associative function and a neutral \"zero value.\"\n",
      " |      \n",
      " |      The function C{op(t1, t2)} is allowed to modify C{t1} and return it\n",
      " |      as its result value to avoid object allocation; however, it should not\n",
      " |      modify C{t2}.\n",
      " |      \n",
      " |      This behaves somewhat differently from fold operations implemented\n",
      " |      for non-distributed collections in functional languages like Scala.\n",
      " |      This fold operation may be applied to partitions individually, and then\n",
      " |      fold those results into the final result, rather than apply the fold\n",
      " |      to each element sequentially in some defined ordering. For functions\n",
      " |      that are not commutative, the result may differ from that of a fold\n",
      " |      applied to a non-distributed collection.\n",
      " |      \n",
      " |      >>> from operator import add\n",
      " |      >>> sc.parallelize([1, 2, 3, 4, 5]).fold(0, add)\n",
      " |      15\n",
      " |  \n",
      " |  foldByKey(self, zeroValue, func, numPartitions=None, partitionFunc=<function portable_hash at 0x7f4817231a60>)\n",
      " |      Merge the values for each key using an associative function \"func\"\n",
      " |      and a neutral \"zeroValue\" which may be added to the result an\n",
      " |      arbitrary number of times, and must not change the result\n",
      " |      (e.g., 0 for addition, or 1 for multiplication.).\n",
      " |      \n",
      " |      >>> rdd = sc.parallelize([(\"a\", 1), (\"b\", 1), (\"a\", 1)])\n",
      " |      >>> from operator import add\n",
      " |      >>> sorted(rdd.foldByKey(0, add).collect())\n",
      " |      [('a', 2), ('b', 1)]\n",
      " |  \n",
      " |  foreach(self, f)\n",
      " |      Applies a function to all elements of this RDD.\n",
      " |      \n",
      " |      >>> def f(x): print(x)\n",
      " |      >>> sc.parallelize([1, 2, 3, 4, 5]).foreach(f)\n",
      " |  \n",
      " |  foreachPartition(self, f)\n",
      " |      Applies a function to each partition of this RDD.\n",
      " |      \n",
      " |      >>> def f(iterator):\n",
      " |      ...     for x in iterator:\n",
      " |      ...          print(x)\n",
      " |      >>> sc.parallelize([1, 2, 3, 4, 5]).foreachPartition(f)\n",
      " |  \n",
      " |  fullOuterJoin(self, other, numPartitions=None)\n",
      " |      Perform a right outer join of C{self} and C{other}.\n",
      " |      \n",
      " |      For each element (k, v) in C{self}, the resulting RDD will either\n",
      " |      contain all pairs (k, (v, w)) for w in C{other}, or the pair\n",
      " |      (k, (v, None)) if no elements in C{other} have key k.\n",
      " |      \n",
      " |      Similarly, for each element (k, w) in C{other}, the resulting RDD will\n",
      " |      either contain all pairs (k, (v, w)) for v in C{self}, or the pair\n",
      " |      (k, (None, w)) if no elements in C{self} have key k.\n",
      " |      \n",
      " |      Hash-partitions the resulting RDD into the given number of partitions.\n",
      " |      \n",
      " |      >>> x = sc.parallelize([(\"a\", 1), (\"b\", 4)])\n",
      " |      >>> y = sc.parallelize([(\"a\", 2), (\"c\", 8)])\n",
      " |      >>> sorted(x.fullOuterJoin(y).collect())\n",
      " |      [('a', (1, 2)), ('b', (4, None)), ('c', (None, 8))]\n",
      " |  \n",
      " |  getCheckpointFile(self)\n",
      " |      Gets the name of the file to which this RDD was checkpointed\n",
      " |      \n",
      " |      Not defined if RDD is checkpointed locally.\n",
      " |  \n",
      " |  getNumPartitions(self)\n",
      " |      Returns the number of partitions in RDD\n",
      " |      \n",
      " |      >>> rdd = sc.parallelize([1, 2, 3, 4], 2)\n",
      " |      >>> rdd.getNumPartitions()\n",
      " |      2\n",
      " |  \n",
      " |  getStorageLevel(self)\n",
      " |      Get the RDD's current storage level.\n",
      " |      \n",
      " |      >>> rdd1 = sc.parallelize([1,2])\n",
      " |      >>> rdd1.getStorageLevel()\n",
      " |      StorageLevel(False, False, False, False, 1)\n",
      " |      >>> print(rdd1.getStorageLevel())\n",
      " |      Serialized 1x Replicated\n",
      " |  \n",
      " |  glom(self)\n",
      " |      Return an RDD created by coalescing all elements within each partition\n",
      " |      into a list.\n",
      " |      \n",
      " |      >>> rdd = sc.parallelize([1, 2, 3, 4], 2)\n",
      " |      >>> sorted(rdd.glom().collect())\n",
      " |      [[1, 2], [3, 4]]\n",
      " |  \n",
      " |  groupBy(self, f, numPartitions=None, partitionFunc=<function portable_hash at 0x7f4817231a60>)\n",
      " |      Return an RDD of grouped items.\n",
      " |      \n",
      " |      >>> rdd = sc.parallelize([1, 1, 2, 3, 5, 8])\n",
      " |      >>> result = rdd.groupBy(lambda x: x % 2).collect()\n",
      " |      >>> sorted([(x, sorted(y)) for (x, y) in result])\n",
      " |      [(0, [2, 8]), (1, [1, 1, 3, 5])]\n",
      " |  \n",
      " |  groupByKey(self, numPartitions=None, partitionFunc=<function portable_hash at 0x7f4817231a60>)\n",
      " |      Group the values for each key in the RDD into a single sequence.\n",
      " |      Hash-partitions the resulting RDD with numPartitions partitions.\n",
      " |      \n",
      " |      .. note:: If you are grouping in order to perform an aggregation (such as a\n",
      " |          sum or average) over each key, using reduceByKey or aggregateByKey will\n",
      " |          provide much better performance.\n",
      " |      \n",
      " |      >>> rdd = sc.parallelize([(\"a\", 1), (\"b\", 1), (\"a\", 1)])\n",
      " |      >>> sorted(rdd.groupByKey().mapValues(len).collect())\n",
      " |      [('a', 2), ('b', 1)]\n",
      " |      >>> sorted(rdd.groupByKey().mapValues(list).collect())\n",
      " |      [('a', [1, 1]), ('b', [1])]\n",
      " |  \n",
      " |  groupWith(self, other, *others)\n",
      " |      Alias for cogroup but with support for multiple RDDs.\n",
      " |      \n",
      " |      >>> w = sc.parallelize([(\"a\", 5), (\"b\", 6)])\n",
      " |      >>> x = sc.parallelize([(\"a\", 1), (\"b\", 4)])\n",
      " |      >>> y = sc.parallelize([(\"a\", 2)])\n",
      " |      >>> z = sc.parallelize([(\"b\", 42)])\n",
      " |      >>> [(x, tuple(map(list, y))) for x, y in sorted(list(w.groupWith(x, y, z).collect()))]\n",
      " |      [('a', ([5], [1], [2], [])), ('b', ([6], [4], [], [42]))]\n",
      " |  \n",
      " |  histogram(self, buckets)\n",
      " |      Compute a histogram using the provided buckets. The buckets\n",
      " |      are all open to the right except for the last which is closed.\n",
      " |      e.g. [1,10,20,50] means the buckets are [1,10) [10,20) [20,50],\n",
      " |      which means 1<=x<10, 10<=x<20, 20<=x<=50. And on the input of 1\n",
      " |      and 50 we would have a histogram of 1,0,1.\n",
      " |      \n",
      " |      If your histogram is evenly spaced (e.g. [0, 10, 20, 30]),\n",
      " |      this can be switched from an O(log n) inseration to O(1) per\n",
      " |      element (where n is the number of buckets).\n",
      " |      \n",
      " |      Buckets must be sorted, not contain any duplicates, and have\n",
      " |      at least two elements.\n",
      " |      \n",
      " |      If `buckets` is a number, it will generate buckets which are\n",
      " |      evenly spaced between the minimum and maximum of the RDD. For\n",
      " |      example, if the min value is 0 and the max is 100, given `buckets`\n",
      " |      as 2, the resulting buckets will be [0,50) [50,100]. `buckets` must\n",
      " |      be at least 1. An exception is raised if the RDD contains infinity.\n",
      " |      If the elements in the RDD do not vary (max == min), a single bucket\n",
      " |      will be used.\n",
      " |      \n",
      " |      The return value is a tuple of buckets and histogram.\n",
      " |      \n",
      " |      >>> rdd = sc.parallelize(range(51))\n",
      " |      >>> rdd.histogram(2)\n",
      " |      ([0, 25, 50], [25, 26])\n",
      " |      >>> rdd.histogram([0, 5, 25, 50])\n",
      " |      ([0, 5, 25, 50], [5, 20, 26])\n",
      " |      >>> rdd.histogram([0, 15, 30, 45, 60])  # evenly spaced buckets\n",
      " |      ([0, 15, 30, 45, 60], [15, 15, 15, 6])\n",
      " |      >>> rdd = sc.parallelize([\"ab\", \"ac\", \"b\", \"bd\", \"ef\"])\n",
      " |      >>> rdd.histogram((\"a\", \"b\", \"c\"))\n",
      " |      (('a', 'b', 'c'), [2, 2])\n",
      " |  \n",
      " |  id(self)\n",
      " |      A unique ID for this RDD (within its SparkContext).\n",
      " |  \n",
      " |  intersection(self, other)\n",
      " |      Return the intersection of this RDD and another one. The output will\n",
      " |      not contain any duplicate elements, even if the input RDDs did.\n",
      " |      \n",
      " |      .. note:: This method performs a shuffle internally.\n",
      " |      \n",
      " |      >>> rdd1 = sc.parallelize([1, 10, 2, 3, 4, 5])\n",
      " |      >>> rdd2 = sc.parallelize([1, 6, 2, 3, 7, 8])\n",
      " |      >>> rdd1.intersection(rdd2).collect()\n",
      " |      [1, 2, 3]\n",
      " |  \n",
      " |  isCheckpointed(self)\n",
      " |      Return whether this RDD is checkpointed and materialized, either reliably or locally.\n",
      " |  \n",
      " |  isEmpty(self)\n",
      " |      Returns true if and only if the RDD contains no elements at all.\n",
      " |      \n",
      " |      .. note:: an RDD may be empty even when it has at least 1 partition.\n",
      " |      \n",
      " |      >>> sc.parallelize([]).isEmpty()\n",
      " |      True\n",
      " |      >>> sc.parallelize([1]).isEmpty()\n",
      " |      False\n",
      " |  \n",
      " |  isLocallyCheckpointed(self)\n",
      " |      Return whether this RDD is marked for local checkpointing.\n",
      " |      \n",
      " |      Exposed for testing.\n",
      " |  \n",
      " |  join(self, other, numPartitions=None)\n",
      " |      Return an RDD containing all pairs of elements with matching keys in\n",
      " |      C{self} and C{other}.\n",
      " |      \n",
      " |      Each pair of elements will be returned as a (k, (v1, v2)) tuple, where\n",
      " |      (k, v1) is in C{self} and (k, v2) is in C{other}.\n",
      " |      \n",
      " |      Performs a hash join across the cluster.\n",
      " |      \n",
      " |      >>> x = sc.parallelize([(\"a\", 1), (\"b\", 4)])\n",
      " |      >>> y = sc.parallelize([(\"a\", 2), (\"a\", 3)])\n",
      " |      >>> sorted(x.join(y).collect())\n",
      " |      [('a', (1, 2)), ('a', (1, 3))]\n",
      " |  \n",
      " |  keyBy(self, f)\n",
      " |      Creates tuples of the elements in this RDD by applying C{f}.\n",
      " |      \n",
      " |      >>> x = sc.parallelize(range(0,3)).keyBy(lambda x: x*x)\n",
      " |      >>> y = sc.parallelize(zip(range(0,5), range(0,5)))\n",
      " |      >>> [(x, list(map(list, y))) for x, y in sorted(x.cogroup(y).collect())]\n",
      " |      [(0, [[0], [0]]), (1, [[1], [1]]), (2, [[], [2]]), (3, [[], [3]]), (4, [[2], [4]])]\n",
      " |  \n",
      " |  keys(self)\n",
      " |      Return an RDD with the keys of each tuple.\n",
      " |      \n",
      " |      >>> m = sc.parallelize([(1, 2), (3, 4)]).keys()\n",
      " |      >>> m.collect()\n",
      " |      [1, 3]\n",
      " |  \n",
      " |  leftOuterJoin(self, other, numPartitions=None)\n",
      " |      Perform a left outer join of C{self} and C{other}.\n",
      " |      \n",
      " |      For each element (k, v) in C{self}, the resulting RDD will either\n",
      " |      contain all pairs (k, (v, w)) for w in C{other}, or the pair\n",
      " |      (k, (v, None)) if no elements in C{other} have key k.\n",
      " |      \n",
      " |      Hash-partitions the resulting RDD into the given number of partitions.\n",
      " |      \n",
      " |      >>> x = sc.parallelize([(\"a\", 1), (\"b\", 4)])\n",
      " |      >>> y = sc.parallelize([(\"a\", 2)])\n",
      " |      >>> sorted(x.leftOuterJoin(y).collect())\n",
      " |      [('a', (1, 2)), ('b', (4, None))]\n",
      " |  \n",
      " |  localCheckpoint(self)\n",
      " |      Mark this RDD for local checkpointing using Spark's existing caching layer.\n",
      " |      \n",
      " |      This method is for users who wish to truncate RDD lineages while skipping the expensive\n",
      " |      step of replicating the materialized data in a reliable distributed file system. This is\n",
      " |      useful for RDDs with long lineages that need to be truncated periodically (e.g. GraphX).\n",
      " |      \n",
      " |      Local checkpointing sacrifices fault-tolerance for performance. In particular, checkpointed\n",
      " |      data is written to ephemeral local storage in the executors instead of to a reliable,\n",
      " |      fault-tolerant storage. The effect is that if an executor fails during the computation,\n",
      " |      the checkpointed data may no longer be accessible, causing an irrecoverable job failure.\n",
      " |      \n",
      " |      This is NOT safe to use with dynamic allocation, which removes executors along\n",
      " |      with their cached blocks. If you must use both features, you are advised to set\n",
      " |      L{spark.dynamicAllocation.cachedExecutorIdleTimeout} to a high value.\n",
      " |      \n",
      " |      The checkpoint directory set through L{SparkContext.setCheckpointDir()} is not used.\n",
      " |  \n",
      " |  lookup(self, key)\n",
      " |      Return the list of values in the RDD for key `key`. This operation\n",
      " |      is done efficiently if the RDD has a known partitioner by only\n",
      " |      searching the partition that the key maps to.\n",
      " |      \n",
      " |      >>> l = range(1000)\n",
      " |      >>> rdd = sc.parallelize(zip(l, l), 10)\n",
      " |      >>> rdd.lookup(42)  # slow\n",
      " |      [42]\n",
      " |      >>> sorted = rdd.sortByKey()\n",
      " |      >>> sorted.lookup(42)  # fast\n",
      " |      [42]\n",
      " |      >>> sorted.lookup(1024)\n",
      " |      []\n",
      " |      >>> rdd2 = sc.parallelize([(('a', 'b'), 'c')]).groupByKey()\n",
      " |      >>> list(rdd2.lookup(('a', 'b'))[0])\n",
      " |      ['c']\n",
      " |  \n",
      " |  map(self, f, preservesPartitioning=False)\n",
      " |      Return a new RDD by applying a function to each element of this RDD.\n",
      " |      \n",
      " |      >>> rdd = sc.parallelize([\"b\", \"a\", \"c\"])\n",
      " |      >>> sorted(rdd.map(lambda x: (x, 1)).collect())\n",
      " |      [('a', 1), ('b', 1), ('c', 1)]\n",
      " |  \n",
      " |  mapPartitions(self, f, preservesPartitioning=False)\n",
      " |      Return a new RDD by applying a function to each partition of this RDD.\n",
      " |      \n",
      " |      >>> rdd = sc.parallelize([1, 2, 3, 4], 2)\n",
      " |      >>> def f(iterator): yield sum(iterator)\n",
      " |      >>> rdd.mapPartitions(f).collect()\n",
      " |      [3, 7]\n",
      " |  \n",
      " |  mapPartitionsWithIndex(self, f, preservesPartitioning=False)\n",
      " |      Return a new RDD by applying a function to each partition of this RDD,\n",
      " |      while tracking the index of the original partition.\n",
      " |      \n",
      " |      >>> rdd = sc.parallelize([1, 2, 3, 4], 4)\n",
      " |      >>> def f(splitIndex, iterator): yield splitIndex\n",
      " |      >>> rdd.mapPartitionsWithIndex(f).sum()\n",
      " |      6\n",
      " |  \n",
      " |  mapPartitionsWithSplit(self, f, preservesPartitioning=False)\n",
      " |      Deprecated: use mapPartitionsWithIndex instead.\n",
      " |      \n",
      " |      Return a new RDD by applying a function to each partition of this RDD,\n",
      " |      while tracking the index of the original partition.\n",
      " |      \n",
      " |      >>> rdd = sc.parallelize([1, 2, 3, 4], 4)\n",
      " |      >>> def f(splitIndex, iterator): yield splitIndex\n",
      " |      >>> rdd.mapPartitionsWithSplit(f).sum()\n",
      " |      6\n",
      " |  \n",
      " |  mapValues(self, f)\n",
      " |      Pass each value in the key-value pair RDD through a map function\n",
      " |      without changing the keys; this also retains the original RDD's\n",
      " |      partitioning.\n",
      " |      \n",
      " |      >>> x = sc.parallelize([(\"a\", [\"apple\", \"banana\", \"lemon\"]), (\"b\", [\"grapes\"])])\n",
      " |      >>> def f(x): return len(x)\n",
      " |      >>> x.mapValues(f).collect()\n",
      " |      [('a', 3), ('b', 1)]\n",
      " |  \n",
      " |  max(self, key=None)\n",
      " |      Find the maximum item in this RDD.\n",
      " |      \n",
      " |      :param key: A function used to generate key for comparing\n",
      " |      \n",
      " |      >>> rdd = sc.parallelize([1.0, 5.0, 43.0, 10.0])\n",
      " |      >>> rdd.max()\n",
      " |      43.0\n",
      " |      >>> rdd.max(key=str)\n",
      " |      5.0\n",
      " |  \n",
      " |  mean(self)\n",
      " |      Compute the mean of this RDD's elements.\n",
      " |      \n",
      " |      >>> sc.parallelize([1, 2, 3]).mean()\n",
      " |      2.0\n",
      " |  \n",
      " |  meanApprox(self, timeout, confidence=0.95)\n",
      " |      .. note:: Experimental\n",
      " |      \n",
      " |      Approximate operation to return the mean within a timeout\n",
      " |      or meet the confidence.\n",
      " |      \n",
      " |      >>> rdd = sc.parallelize(range(1000), 10)\n",
      " |      >>> r = sum(range(1000)) / 1000.0\n",
      " |      >>> abs(rdd.meanApprox(1000) - r) / r < 0.05\n",
      " |      True\n",
      " |  \n",
      " |  min(self, key=None)\n",
      " |      Find the minimum item in this RDD.\n",
      " |      \n",
      " |      :param key: A function used to generate key for comparing\n",
      " |      \n",
      " |      >>> rdd = sc.parallelize([2.0, 5.0, 43.0, 10.0])\n",
      " |      >>> rdd.min()\n",
      " |      2.0\n",
      " |      >>> rdd.min(key=str)\n",
      " |      10.0\n",
      " |  \n",
      " |  name(self)\n",
      " |      Return the name of this RDD.\n",
      " |  \n",
      " |  partitionBy(self, numPartitions, partitionFunc=<function portable_hash at 0x7f4817231a60>)\n",
      " |      Return a copy of the RDD partitioned using the specified partitioner.\n",
      " |      \n",
      " |      >>> pairs = sc.parallelize([1, 2, 3, 4, 2, 4, 1]).map(lambda x: (x, x))\n",
      " |      >>> sets = pairs.partitionBy(2).glom().collect()\n",
      " |      >>> len(set(sets[0]).intersection(set(sets[1])))\n",
      " |      0\n",
      " |  \n",
      " |  persist(self, storageLevel=StorageLevel(False, True, False, False, 1))\n",
      " |      Set this RDD's storage level to persist its values across operations\n",
      " |      after the first time it is computed. This can only be used to assign\n",
      " |      a new storage level if the RDD does not have a storage level set yet.\n",
      " |      If no storage level is specified defaults to (C{MEMORY_ONLY}).\n",
      " |      \n",
      " |      >>> rdd = sc.parallelize([\"b\", \"a\", \"c\"])\n",
      " |      >>> rdd.persist().is_cached\n",
      " |      True\n",
      " |  \n",
      " |  pipe(self, command, env=None, checkCode=False)\n",
      " |      Return an RDD created by piping elements to a forked external process.\n",
      " |      \n",
      " |      >>> sc.parallelize(['1', '2', '', '3']).pipe('cat').collect()\n",
      " |      ['1', '2', '', '3']\n",
      " |      \n",
      " |      :param checkCode: whether or not to check the return value of the shell command.\n",
      " |  \n",
      " |  randomSplit(self, weights, seed=None)\n",
      " |      Randomly splits this RDD with the provided weights.\n",
      " |      \n",
      " |      :param weights: weights for splits, will be normalized if they don't sum to 1\n",
      " |      :param seed: random seed\n",
      " |      :return: split RDDs in a list\n",
      " |      \n",
      " |      >>> rdd = sc.parallelize(range(500), 1)\n",
      " |      >>> rdd1, rdd2 = rdd.randomSplit([2, 3], 17)\n",
      " |      >>> len(rdd1.collect() + rdd2.collect())\n",
      " |      500\n",
      " |      >>> 150 < rdd1.count() < 250\n",
      " |      True\n",
      " |      >>> 250 < rdd2.count() < 350\n",
      " |      True\n",
      " |  \n",
      " |  reduce(self, f)\n",
      " |      Reduces the elements of this RDD using the specified commutative and\n",
      " |      associative binary operator. Currently reduces partitions locally.\n",
      " |      \n",
      " |      >>> from operator import add\n",
      " |      >>> sc.parallelize([1, 2, 3, 4, 5]).reduce(add)\n",
      " |      15\n",
      " |      >>> sc.parallelize((2 for _ in range(10))).map(lambda x: 1).cache().reduce(add)\n",
      " |      10\n",
      " |      >>> sc.parallelize([]).reduce(add)\n",
      " |      Traceback (most recent call last):\n",
      " |          ...\n",
      " |      ValueError: Can not reduce() empty RDD\n",
      " |  \n",
      " |  reduceByKey(self, func, numPartitions=None, partitionFunc=<function portable_hash at 0x7f4817231a60>)\n",
      " |      Merge the values for each key using an associative and commutative reduce function.\n",
      " |      \n",
      " |      This will also perform the merging locally on each mapper before\n",
      " |      sending results to a reducer, similarly to a \"combiner\" in MapReduce.\n",
      " |      \n",
      " |      Output will be partitioned with C{numPartitions} partitions, or\n",
      " |      the default parallelism level if C{numPartitions} is not specified.\n",
      " |      Default partitioner is hash-partition.\n",
      " |      \n",
      " |      >>> from operator import add\n",
      " |      >>> rdd = sc.parallelize([(\"a\", 1), (\"b\", 1), (\"a\", 1)])\n",
      " |      >>> sorted(rdd.reduceByKey(add).collect())\n",
      " |      [('a', 2), ('b', 1)]\n",
      " |  \n",
      " |  reduceByKeyLocally(self, func)\n",
      " |      Merge the values for each key using an associative and commutative reduce function, but\n",
      " |      return the results immediately to the master as a dictionary.\n",
      " |      \n",
      " |      This will also perform the merging locally on each mapper before\n",
      " |      sending results to a reducer, similarly to a \"combiner\" in MapReduce.\n",
      " |      \n",
      " |      >>> from operator import add\n",
      " |      >>> rdd = sc.parallelize([(\"a\", 1), (\"b\", 1), (\"a\", 1)])\n",
      " |      >>> sorted(rdd.reduceByKeyLocally(add).items())\n",
      " |      [('a', 2), ('b', 1)]\n",
      " |  \n",
      " |  repartition(self, numPartitions)\n",
      " |      Return a new RDD that has exactly numPartitions partitions.\n",
      " |      \n",
      " |      Can increase or decrease the level of parallelism in this RDD.\n",
      " |      Internally, this uses a shuffle to redistribute data.\n",
      " |      If you are decreasing the number of partitions in this RDD, consider\n",
      " |      using `coalesce`, which can avoid performing a shuffle.\n",
      " |      \n",
      " |      >>> rdd = sc.parallelize([1,2,3,4,5,6,7], 4)\n",
      " |      >>> sorted(rdd.glom().collect())\n",
      " |      [[1], [2, 3], [4, 5], [6, 7]]\n",
      " |      >>> len(rdd.repartition(2).glom().collect())\n",
      " |      2\n",
      " |      >>> len(rdd.repartition(10).glom().collect())\n",
      " |      10\n",
      " |  \n",
      " |  repartitionAndSortWithinPartitions(self, numPartitions=None, partitionFunc=<function portable_hash at 0x7f4817231a60>, ascending=True, keyfunc=<function RDD.<lambda> at 0x7f48140f2510>)\n",
      " |      Repartition the RDD according to the given partitioner and, within each resulting partition,\n",
      " |      sort records by their keys.\n",
      " |      \n",
      " |      >>> rdd = sc.parallelize([(0, 5), (3, 8), (2, 6), (0, 8), (3, 8), (1, 3)])\n",
      " |      >>> rdd2 = rdd.repartitionAndSortWithinPartitions(2, lambda x: x % 2, 2)\n",
      " |      >>> rdd2.glom().collect()\n",
      " |      [[(0, 5), (0, 8), (2, 6)], [(1, 3), (3, 8), (3, 8)]]\n",
      " |  \n",
      " |  rightOuterJoin(self, other, numPartitions=None)\n",
      " |      Perform a right outer join of C{self} and C{other}.\n",
      " |      \n",
      " |      For each element (k, w) in C{other}, the resulting RDD will either\n",
      " |      contain all pairs (k, (v, w)) for v in this, or the pair (k, (None, w))\n",
      " |      if no elements in C{self} have key k.\n",
      " |      \n",
      " |      Hash-partitions the resulting RDD into the given number of partitions.\n",
      " |      \n",
      " |      >>> x = sc.parallelize([(\"a\", 1), (\"b\", 4)])\n",
      " |      >>> y = sc.parallelize([(\"a\", 2)])\n",
      " |      >>> sorted(y.rightOuterJoin(x).collect())\n",
      " |      [('a', (2, 1)), ('b', (None, 4))]\n",
      " |  \n",
      " |  sample(self, withReplacement, fraction, seed=None)\n",
      " |      Return a sampled subset of this RDD.\n",
      " |      \n",
      " |      :param withReplacement: can elements be sampled multiple times (replaced when sampled out)\n",
      " |      :param fraction: expected size of the sample as a fraction of this RDD's size\n",
      " |          without replacement: probability that each element is chosen; fraction must be [0, 1]\n",
      " |          with replacement: expected number of times each element is chosen; fraction must be >= 0\n",
      " |      :param seed: seed for the random number generator\n",
      " |      \n",
      " |      .. note:: This is not guaranteed to provide exactly the fraction specified of the total\n",
      " |          count of the given :class:`DataFrame`.\n",
      " |      \n",
      " |      >>> rdd = sc.parallelize(range(100), 4)\n",
      " |      >>> 6 <= rdd.sample(False, 0.1, 81).count() <= 14\n",
      " |      True\n",
      " |  \n",
      " |  sampleByKey(self, withReplacement, fractions, seed=None)\n",
      " |      Return a subset of this RDD sampled by key (via stratified sampling).\n",
      " |      Create a sample of this RDD using variable sampling rates for\n",
      " |      different keys as specified by fractions, a key to sampling rate map.\n",
      " |      \n",
      " |      >>> fractions = {\"a\": 0.2, \"b\": 0.1}\n",
      " |      >>> rdd = sc.parallelize(fractions.keys()).cartesian(sc.parallelize(range(0, 1000)))\n",
      " |      >>> sample = dict(rdd.sampleByKey(False, fractions, 2).groupByKey().collect())\n",
      " |      >>> 100 < len(sample[\"a\"]) < 300 and 50 < len(sample[\"b\"]) < 150\n",
      " |      True\n",
      " |      >>> max(sample[\"a\"]) <= 999 and min(sample[\"a\"]) >= 0\n",
      " |      True\n",
      " |      >>> max(sample[\"b\"]) <= 999 and min(sample[\"b\"]) >= 0\n",
      " |      True\n",
      " |  \n",
      " |  sampleStdev(self)\n",
      " |      Compute the sample standard deviation of this RDD's elements (which\n",
      " |      corrects for bias in estimating the standard deviation by dividing by\n",
      " |      N-1 instead of N).\n",
      " |      \n",
      " |      >>> sc.parallelize([1, 2, 3]).sampleStdev()\n",
      " |      1.0\n",
      " |  \n",
      " |  sampleVariance(self)\n",
      " |      Compute the sample variance of this RDD's elements (which corrects\n",
      " |      for bias in estimating the variance by dividing by N-1 instead of N).\n",
      " |      \n",
      " |      >>> sc.parallelize([1, 2, 3]).sampleVariance()\n",
      " |      1.0\n",
      " |  \n",
      " |  saveAsHadoopDataset(self, conf, keyConverter=None, valueConverter=None)\n",
      " |      Output a Python RDD of key-value pairs (of form C{RDD[(K, V)]}) to any Hadoop file\n",
      " |      system, using the old Hadoop OutputFormat API (mapred package). Keys/values are\n",
      " |      converted for output using either user specified converters or, by default,\n",
      " |      L{org.apache.spark.api.python.JavaToWritableConverter}.\n",
      " |      \n",
      " |      :param conf: Hadoop job configuration, passed in as a dict\n",
      " |      :param keyConverter: (None by default)\n",
      " |      :param valueConverter: (None by default)\n",
      " |  \n",
      " |  saveAsHadoopFile(self, path, outputFormatClass, keyClass=None, valueClass=None, keyConverter=None, valueConverter=None, conf=None, compressionCodecClass=None)\n",
      " |      Output a Python RDD of key-value pairs (of form C{RDD[(K, V)]}) to any Hadoop file\n",
      " |      system, using the old Hadoop OutputFormat API (mapred package). Key and value types\n",
      " |      will be inferred if not specified. Keys and values are converted for output using either\n",
      " |      user specified converters or L{org.apache.spark.api.python.JavaToWritableConverter}. The\n",
      " |      C{conf} is applied on top of the base Hadoop conf associated with the SparkContext\n",
      " |      of this RDD to create a merged Hadoop MapReduce job configuration for saving the data.\n",
      " |      \n",
      " |      :param path: path to Hadoop file\n",
      " |      :param outputFormatClass: fully qualified classname of Hadoop OutputFormat\n",
      " |             (e.g. \"org.apache.hadoop.mapred.SequenceFileOutputFormat\")\n",
      " |      :param keyClass: fully qualified classname of key Writable class\n",
      " |             (e.g. \"org.apache.hadoop.io.IntWritable\", None by default)\n",
      " |      :param valueClass: fully qualified classname of value Writable class\n",
      " |             (e.g. \"org.apache.hadoop.io.Text\", None by default)\n",
      " |      :param keyConverter: (None by default)\n",
      " |      :param valueConverter: (None by default)\n",
      " |      :param conf: (None by default)\n",
      " |      :param compressionCodecClass: (None by default)\n",
      " |  \n",
      " |  saveAsNewAPIHadoopDataset(self, conf, keyConverter=None, valueConverter=None)\n",
      " |      Output a Python RDD of key-value pairs (of form C{RDD[(K, V)]}) to any Hadoop file\n",
      " |      system, using the new Hadoop OutputFormat API (mapreduce package). Keys/values are\n",
      " |      converted for output using either user specified converters or, by default,\n",
      " |      L{org.apache.spark.api.python.JavaToWritableConverter}.\n",
      " |      \n",
      " |      :param conf: Hadoop job configuration, passed in as a dict\n",
      " |      :param keyConverter: (None by default)\n",
      " |      :param valueConverter: (None by default)\n",
      " |  \n",
      " |  saveAsNewAPIHadoopFile(self, path, outputFormatClass, keyClass=None, valueClass=None, keyConverter=None, valueConverter=None, conf=None)\n",
      " |      Output a Python RDD of key-value pairs (of form C{RDD[(K, V)]}) to any Hadoop file\n",
      " |      system, using the new Hadoop OutputFormat API (mapreduce package). Key and value types\n",
      " |      will be inferred if not specified. Keys and values are converted for output using either\n",
      " |      user specified converters or L{org.apache.spark.api.python.JavaToWritableConverter}. The\n",
      " |      C{conf} is applied on top of the base Hadoop conf associated with the SparkContext\n",
      " |      of this RDD to create a merged Hadoop MapReduce job configuration for saving the data.\n",
      " |      \n",
      " |      :param path: path to Hadoop file\n",
      " |      :param outputFormatClass: fully qualified classname of Hadoop OutputFormat\n",
      " |             (e.g. \"org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat\")\n",
      " |      :param keyClass: fully qualified classname of key Writable class\n",
      " |             (e.g. \"org.apache.hadoop.io.IntWritable\", None by default)\n",
      " |      :param valueClass: fully qualified classname of value Writable class\n",
      " |             (e.g. \"org.apache.hadoop.io.Text\", None by default)\n",
      " |      :param keyConverter: (None by default)\n",
      " |      :param valueConverter: (None by default)\n",
      " |      :param conf: Hadoop job configuration, passed in as a dict (None by default)\n",
      " |  \n",
      " |  saveAsPickleFile(self, path, batchSize=10)\n",
      " |      Save this RDD as a SequenceFile of serialized objects. The serializer\n",
      " |      used is L{pyspark.serializers.PickleSerializer}, default batch size\n",
      " |      is 10.\n",
      " |      \n",
      " |      >>> tmpFile = NamedTemporaryFile(delete=True)\n",
      " |      >>> tmpFile.close()\n",
      " |      >>> sc.parallelize([1, 2, 'spark', 'rdd']).saveAsPickleFile(tmpFile.name, 3)\n",
      " |      >>> sorted(sc.pickleFile(tmpFile.name, 5).map(str).collect())\n",
      " |      ['1', '2', 'rdd', 'spark']\n",
      " |  \n",
      " |  saveAsSequenceFile(self, path, compressionCodecClass=None)\n",
      " |      Output a Python RDD of key-value pairs (of form C{RDD[(K, V)]}) to any Hadoop file\n",
      " |      system, using the L{org.apache.hadoop.io.Writable} types that we convert from the\n",
      " |      RDD's key and value types. The mechanism is as follows:\n",
      " |      \n",
      " |          1. Pyrolite is used to convert pickled Python RDD into RDD of Java objects.\n",
      " |          2. Keys and values of this Java RDD are converted to Writables and written out.\n",
      " |      \n",
      " |      :param path: path to sequence file\n",
      " |      :param compressionCodecClass: (None by default)\n",
      " |  \n",
      " |  saveAsTextFile(self, path, compressionCodecClass=None)\n",
      " |      Save this RDD as a text file, using string representations of elements.\n",
      " |      \n",
      " |      @param path: path to text file\n",
      " |      @param compressionCodecClass: (None by default) string i.e.\n",
      " |          \"org.apache.hadoop.io.compress.GzipCodec\"\n",
      " |      \n",
      " |      >>> tempFile = NamedTemporaryFile(delete=True)\n",
      " |      >>> tempFile.close()\n",
      " |      >>> sc.parallelize(range(10)).saveAsTextFile(tempFile.name)\n",
      " |      >>> from fileinput import input\n",
      " |      >>> from glob import glob\n",
      " |      >>> ''.join(sorted(input(glob(tempFile.name + \"/part-0000*\"))))\n",
      " |      '0\\n1\\n2\\n3\\n4\\n5\\n6\\n7\\n8\\n9\\n'\n",
      " |      \n",
      " |      Empty lines are tolerated when saving to text files.\n",
      " |      \n",
      " |      >>> tempFile2 = NamedTemporaryFile(delete=True)\n",
      " |      >>> tempFile2.close()\n",
      " |      >>> sc.parallelize(['', 'foo', '', 'bar', '']).saveAsTextFile(tempFile2.name)\n",
      " |      >>> ''.join(sorted(input(glob(tempFile2.name + \"/part-0000*\"))))\n",
      " |      '\\n\\n\\nbar\\nfoo\\n'\n",
      " |      \n",
      " |      Using compressionCodecClass\n",
      " |      \n",
      " |      >>> tempFile3 = NamedTemporaryFile(delete=True)\n",
      " |      >>> tempFile3.close()\n",
      " |      >>> codec = \"org.apache.hadoop.io.compress.GzipCodec\"\n",
      " |      >>> sc.parallelize(['foo', 'bar']).saveAsTextFile(tempFile3.name, codec)\n",
      " |      >>> from fileinput import input, hook_compressed\n",
      " |      >>> result = sorted(input(glob(tempFile3.name + \"/part*.gz\"), openhook=hook_compressed))\n",
      " |      >>> b''.join(result).decode('utf-8')\n",
      " |      'bar\\nfoo\\n'\n",
      " |  \n",
      " |  setName(self, name)\n",
      " |      Assign a name to this RDD.\n",
      " |      \n",
      " |      >>> rdd1 = sc.parallelize([1, 2])\n",
      " |      >>> rdd1.setName('RDD1').name()\n",
      " |      'RDD1'\n",
      " |  \n",
      " |  sortBy(self, keyfunc, ascending=True, numPartitions=None)\n",
      " |      Sorts this RDD by the given keyfunc\n",
      " |      \n",
      " |      >>> tmp = [('a', 1), ('b', 2), ('1', 3), ('d', 4), ('2', 5)]\n",
      " |      >>> sc.parallelize(tmp).sortBy(lambda x: x[0]).collect()\n",
      " |      [('1', 3), ('2', 5), ('a', 1), ('b', 2), ('d', 4)]\n",
      " |      >>> sc.parallelize(tmp).sortBy(lambda x: x[1]).collect()\n",
      " |      [('a', 1), ('b', 2), ('1', 3), ('d', 4), ('2', 5)]\n",
      " |  \n",
      " |  sortByKey(self, ascending=True, numPartitions=None, keyfunc=<function RDD.<lambda> at 0x7f48140f2620>)\n",
      " |      Sorts this RDD, which is assumed to consist of (key, value) pairs.\n",
      " |      # noqa\n",
      " |      \n",
      " |      >>> tmp = [('a', 1), ('b', 2), ('1', 3), ('d', 4), ('2', 5)]\n",
      " |      >>> sc.parallelize(tmp).sortByKey().first()\n",
      " |      ('1', 3)\n",
      " |      >>> sc.parallelize(tmp).sortByKey(True, 1).collect()\n",
      " |      [('1', 3), ('2', 5), ('a', 1), ('b', 2), ('d', 4)]\n",
      " |      >>> sc.parallelize(tmp).sortByKey(True, 2).collect()\n",
      " |      [('1', 3), ('2', 5), ('a', 1), ('b', 2), ('d', 4)]\n",
      " |      >>> tmp2 = [('Mary', 1), ('had', 2), ('a', 3), ('little', 4), ('lamb', 5)]\n",
      " |      >>> tmp2.extend([('whose', 6), ('fleece', 7), ('was', 8), ('white', 9)])\n",
      " |      >>> sc.parallelize(tmp2).sortByKey(True, 3, keyfunc=lambda k: k.lower()).collect()\n",
      " |      [('a', 3), ('fleece', 7), ('had', 2), ('lamb', 5),...('white', 9), ('whose', 6)]\n",
      " |  \n",
      " |  stats(self)\n",
      " |      Return a L{StatCounter} object that captures the mean, variance\n",
      " |      and count of the RDD's elements in one operation.\n",
      " |  \n",
      " |  stdev(self)\n",
      " |      Compute the standard deviation of this RDD's elements.\n",
      " |      \n",
      " |      >>> sc.parallelize([1, 2, 3]).stdev()\n",
      " |      0.816...\n",
      " |  \n",
      " |  subtract(self, other, numPartitions=None)\n",
      " |      Return each value in C{self} that is not contained in C{other}.\n",
      " |      \n",
      " |      >>> x = sc.parallelize([(\"a\", 1), (\"b\", 4), (\"b\", 5), (\"a\", 3)])\n",
      " |      >>> y = sc.parallelize([(\"a\", 3), (\"c\", None)])\n",
      " |      >>> sorted(x.subtract(y).collect())\n",
      " |      [('a', 1), ('b', 4), ('b', 5)]\n",
      " |  \n",
      " |  subtractByKey(self, other, numPartitions=None)\n",
      " |      Return each (key, value) pair in C{self} that has no pair with matching\n",
      " |      key in C{other}.\n",
      " |      \n",
      " |      >>> x = sc.parallelize([(\"a\", 1), (\"b\", 4), (\"b\", 5), (\"a\", 2)])\n",
      " |      >>> y = sc.parallelize([(\"a\", 3), (\"c\", None)])\n",
      " |      >>> sorted(x.subtractByKey(y).collect())\n",
      " |      [('b', 4), ('b', 5)]\n",
      " |  \n",
      " |  sum(self)\n",
      " |      Add up the elements in this RDD.\n",
      " |      \n",
      " |      >>> sc.parallelize([1.0, 2.0, 3.0]).sum()\n",
      " |      6.0\n",
      " |  \n",
      " |  sumApprox(self, timeout, confidence=0.95)\n",
      " |      .. note:: Experimental\n",
      " |      \n",
      " |      Approximate operation to return the sum within a timeout\n",
      " |      or meet the confidence.\n",
      " |      \n",
      " |      >>> rdd = sc.parallelize(range(1000), 10)\n",
      " |      >>> r = sum(range(1000))\n",
      " |      >>> abs(rdd.sumApprox(1000) - r) / r < 0.05\n",
      " |      True\n",
      " |  \n",
      " |  take(self, num)\n",
      " |      Take the first num elements of the RDD.\n",
      " |      \n",
      " |      It works by first scanning one partition, and use the results from\n",
      " |      that partition to estimate the number of additional partitions needed\n",
      " |      to satisfy the limit.\n",
      " |      \n",
      " |      Translated from the Scala implementation in RDD#take().\n",
      " |      \n",
      " |      .. note:: this method should only be used if the resulting array is expected\n",
      " |          to be small, as all the data is loaded into the driver's memory.\n",
      " |      \n",
      " |      >>> sc.parallelize([2, 3, 4, 5, 6]).cache().take(2)\n",
      " |      [2, 3]\n",
      " |      >>> sc.parallelize([2, 3, 4, 5, 6]).take(10)\n",
      " |      [2, 3, 4, 5, 6]\n",
      " |      >>> sc.parallelize(range(100), 100).filter(lambda x: x > 90).take(3)\n",
      " |      [91, 92, 93]\n",
      " |  \n",
      " |  takeOrdered(self, num, key=None)\n",
      " |      Get the N elements from an RDD ordered in ascending order or as\n",
      " |      specified by the optional key function.\n",
      " |      \n",
      " |      .. note:: this method should only be used if the resulting array is expected\n",
      " |          to be small, as all the data is loaded into the driver's memory.\n",
      " |      \n",
      " |      >>> sc.parallelize([10, 1, 2, 9, 3, 4, 5, 6, 7]).takeOrdered(6)\n",
      " |      [1, 2, 3, 4, 5, 6]\n",
      " |      >>> sc.parallelize([10, 1, 2, 9, 3, 4, 5, 6, 7], 2).takeOrdered(6, key=lambda x: -x)\n",
      " |      [10, 9, 7, 6, 5, 4]\n",
      " |  \n",
      " |  takeSample(self, withReplacement, num, seed=None)\n",
      " |      Return a fixed-size sampled subset of this RDD.\n",
      " |      \n",
      " |      .. note:: This method should only be used if the resulting array is expected\n",
      " |          to be small, as all the data is loaded into the driver's memory.\n",
      " |      \n",
      " |      >>> rdd = sc.parallelize(range(0, 10))\n",
      " |      >>> len(rdd.takeSample(True, 20, 1))\n",
      " |      20\n",
      " |      >>> len(rdd.takeSample(False, 5, 2))\n",
      " |      5\n",
      " |      >>> len(rdd.takeSample(False, 15, 3))\n",
      " |      10\n",
      " |  \n",
      " |  toDebugString(self)\n",
      " |      A description of this RDD and its recursive dependencies for debugging.\n",
      " |  \n",
      " |  toLocalIterator(self)\n",
      " |      Return an iterator that contains all of the elements in this RDD.\n",
      " |      The iterator will consume as much memory as the largest partition in this RDD.\n",
      " |      \n",
      " |      >>> rdd = sc.parallelize(range(10))\n",
      " |      >>> [x for x in rdd.toLocalIterator()]\n",
      " |      [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]\n",
      " |  \n",
      " |  top(self, num, key=None)\n",
      " |      Get the top N elements from an RDD.\n",
      " |      \n",
      " |      .. note:: This method should only be used if the resulting array is expected\n",
      " |          to be small, as all the data is loaded into the driver's memory.\n",
      " |      \n",
      " |      .. note:: It returns the list sorted in descending order.\n",
      " |      \n",
      " |      >>> sc.parallelize([10, 4, 2, 12, 3]).top(1)\n",
      " |      [12]\n",
      " |      >>> sc.parallelize([2, 3, 4, 5, 6], 2).top(2)\n",
      " |      [6, 5]\n",
      " |      >>> sc.parallelize([10, 4, 2, 12, 3]).top(3, key=str)\n",
      " |      [4, 3, 2]\n",
      " |  \n",
      " |  treeAggregate(self, zeroValue, seqOp, combOp, depth=2)\n",
      " |      Aggregates the elements of this RDD in a multi-level tree\n",
      " |      pattern.\n",
      " |      \n",
      " |      :param depth: suggested depth of the tree (default: 2)\n",
      " |      \n",
      " |      >>> add = lambda x, y: x + y\n",
      " |      >>> rdd = sc.parallelize([-5, -4, -3, -2, -1, 1, 2, 3, 4], 10)\n",
      " |      >>> rdd.treeAggregate(0, add, add)\n",
      " |      -5\n",
      " |      >>> rdd.treeAggregate(0, add, add, 1)\n",
      " |      -5\n",
      " |      >>> rdd.treeAggregate(0, add, add, 2)\n",
      " |      -5\n",
      " |      >>> rdd.treeAggregate(0, add, add, 5)\n",
      " |      -5\n",
      " |      >>> rdd.treeAggregate(0, add, add, 10)\n",
      " |      -5\n",
      " |  \n",
      " |  treeReduce(self, f, depth=2)\n",
      " |      Reduces the elements of this RDD in a multi-level tree pattern.\n",
      " |      \n",
      " |      :param depth: suggested depth of the tree (default: 2)\n",
      " |      \n",
      " |      >>> add = lambda x, y: x + y\n",
      " |      >>> rdd = sc.parallelize([-5, -4, -3, -2, -1, 1, 2, 3, 4], 10)\n",
      " |      >>> rdd.treeReduce(add)\n",
      " |      -5\n",
      " |      >>> rdd.treeReduce(add, 1)\n",
      " |      -5\n",
      " |      >>> rdd.treeReduce(add, 2)\n",
      " |      -5\n",
      " |      >>> rdd.treeReduce(add, 5)\n",
      " |      -5\n",
      " |      >>> rdd.treeReduce(add, 10)\n",
      " |      -5\n",
      " |  \n",
      " |  union(self, other)\n",
      " |      Return the union of this RDD and another one.\n",
      " |      \n",
      " |      >>> rdd = sc.parallelize([1, 1, 2, 3])\n",
      " |      >>> rdd.union(rdd).collect()\n",
      " |      [1, 1, 2, 3, 1, 1, 2, 3]\n",
      " |  \n",
      " |  unpersist(self)\n",
      " |      Mark the RDD as non-persistent, and remove all blocks for it from\n",
      " |      memory and disk.\n",
      " |  \n",
      " |  values(self)\n",
      " |      Return an RDD with the values of each tuple.\n",
      " |      \n",
      " |      >>> m = sc.parallelize([(1, 2), (3, 4)]).values()\n",
      " |      >>> m.collect()\n",
      " |      [2, 4]\n",
      " |  \n",
      " |  variance(self)\n",
      " |      Compute the variance of this RDD's elements.\n",
      " |      \n",
      " |      >>> sc.parallelize([1, 2, 3]).variance()\n",
      " |      0.666...\n",
      " |  \n",
      " |  zip(self, other)\n",
      " |      Zips this RDD with another one, returning key-value pairs with the\n",
      " |      first element in each RDD second element in each RDD, etc. Assumes\n",
      " |      that the two RDDs have the same number of partitions and the same\n",
      " |      number of elements in each partition (e.g. one was made through\n",
      " |      a map on the other).\n",
      " |      \n",
      " |      >>> x = sc.parallelize(range(0,5))\n",
      " |      >>> y = sc.parallelize(range(1000, 1005))\n",
      " |      >>> x.zip(y).collect()\n",
      " |      [(0, 1000), (1, 1001), (2, 1002), (3, 1003), (4, 1004)]\n",
      " |  \n",
      " |  zipWithIndex(self)\n",
      " |      Zips this RDD with its element indices.\n",
      " |      \n",
      " |      The ordering is first based on the partition index and then the\n",
      " |      ordering of items within each partition. So the first item in\n",
      " |      the first partition gets index 0, and the last item in the last\n",
      " |      partition receives the largest index.\n",
      " |      \n",
      " |      This method needs to trigger a spark job when this RDD contains\n",
      " |      more than one partitions.\n",
      " |      \n",
      " |      >>> sc.parallelize([\"a\", \"b\", \"c\", \"d\"], 3).zipWithIndex().collect()\n",
      " |      [('a', 0), ('b', 1), ('c', 2), ('d', 3)]\n",
      " |  \n",
      " |  zipWithUniqueId(self)\n",
      " |      Zips this RDD with generated unique Long ids.\n",
      " |      \n",
      " |      Items in the kth partition will get ids k, n+k, 2*n+k, ..., where\n",
      " |      n is the number of partitions. So there may exist gaps, but this\n",
      " |      method won't trigger a spark job, which is different from\n",
      " |      L{zipWithIndex}\n",
      " |      \n",
      " |      >>> sc.parallelize([\"a\", \"b\", \"c\", \"d\", \"e\"], 3).zipWithUniqueId().collect()\n",
      " |      [('a', 0), ('b', 1), ('c', 4), ('d', 2), ('e', 5)]\n",
      " |  \n",
      " |  ----------------------------------------------------------------------\n",
      " |  Data descriptors defined here:\n",
      " |  \n",
      " |  __dict__\n",
      " |      dictionary for instance variables (if defined)\n",
      " |  \n",
      " |  __weakref__\n",
      " |      list of weak references to the object (if defined)\n",
      " |  \n",
      " |  context\n",
      " |      The L{SparkContext} that this RDD was created on.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "help(textfileRDD)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}