pytastic code 1

#291
Raw
Author
Anonymous
Created
Nov. 15, 2020, 7:57 a.m.
Expires
Never
Size
1.8 MB
Hits
40
Syntax
Python
from __future__ import unicode_literals

from math import ceil

from .compat import compat_b64decode
from .utils import bytes_to_intlist, intlist_to_bytes

BLOCK_SIZE_BYTES = 16


def aes_ctr_decrypt(data, key, counter):
    """
    Decrypt with aes in counter mode

    @param {int[]} data        cipher
    @param {int[]} key         16/24/32-Byte cipher key
    @param {instance} counter  Instance whose next_value function (@returns {int[]}  16-Byte block)
                               returns the next counter block
    @returns {int[]}           decrypted data
    """
    expanded_key = key_expansion(key)
    block_count = int(ceil(float(len(data)) / BLOCK_SIZE_BYTES))

    decrypted_data = []
    for i in range(block_count):
        counter_block = counter.next_value()
        block = data[i * BLOCK_SIZE_BYTES: (i + 1) * BLOCK_SIZE_BYTES]
        block += [0] * (BLOCK_SIZE_BYTES - len(block))

        cipher_counter_block = aes_encrypt(counter_block, expanded_key)
        decrypted_data += xor(block, cipher_counter_block)
    decrypted_data = decrypted_data[:len(data)]

    return decrypted_data


def aes_cbc_decrypt(data, key, iv):
    """
    Decrypt with aes in CBC mode

    @param {int[]} data        cipher
    @param {int[]} key         16/24/32-Byte cipher key
    @param {int[]} iv          16-Byte IV
    @returns {int[]}           decrypted data
    """
    expanded_key = key_expansion(key)
    block_count = int(ceil(float(len(data)) / BLOCK_SIZE_BYTES))

    decrypted_data = []
    previous_cipher_block = iv
    for i in range(block_count):
        block = data[i * BLOCK_SIZE_BYTES: (i + 1) * BLOCK_SIZE_BYTES]
        block += [0] * (BLOCK_SIZE_BYTES - len(block))

        decrypted_block = aes_decrypt(block, expanded_key)
        decrypted_data += xor(decrypted_block, previous_cipher_block)
        previous_cipher_block = block
    decrypted_data = decrypted_data[:len(data)]

    return decrypted_data


def aes_cbc_encrypt(data, key, iv):
    """
    Encrypt with aes in CBC mode. Using PKCS#7 padding

    @param {int[]} data        cleartext
    @param {int[]} key         16/24/32-Byte cipher key
    @param {int[]} iv          16-Byte IV
    @returns {int[]}           encrypted data
    """
    expanded_key = key_expansion(key)
    block_count = int(ceil(float(len(data)) / BLOCK_SIZE_BYTES))

    encrypted_data = []
    previous_cipher_block = iv
    for i in range(block_count):
        block = data[i * BLOCK_SIZE_BYTES: (i + 1) * BLOCK_SIZE_BYTES]
        remaining_length = BLOCK_SIZE_BYTES - len(block)
        block += [remaining_length] * remaining_length
        mixed_block = xor(block, previous_cipher_block)

        encrypted_block = aes_encrypt(mixed_block, expanded_key)
        encrypted_data += encrypted_block

        previous_cipher_block = encrypted_block

    return encrypted_data


def key_expansion(data):
    """
    Generate key schedule

    @param {int[]} data  16/24/32-Byte cipher key
    @returns {int[]}     176/208/240-Byte expanded key
    """
    data = data[:]  # copy
    rcon_iteration = 1
    key_size_bytes = len(data)
    expanded_key_size_bytes = (key_size_bytes // 4 + 7) * BLOCK_SIZE_BYTES

    while len(data) < expanded_key_size_bytes:
        temp = data[-4:]
        temp = key_schedule_core(temp, rcon_iteration)
        rcon_iteration += 1
        data += xor(temp, data[-key_size_bytes: 4 - key_size_bytes])

        for _ in range(3):
            temp = data[-4:]
            data += xor(temp, data[-key_size_bytes: 4 - key_size_bytes])

        if key_size_bytes == 32:
            temp = data[-4:]
            temp = sub_bytes(temp)
            data += xor(temp, data[-key_size_bytes: 4 - key_size_bytes])

        for _ in range(3 if key_size_bytes == 32 else 2 if key_size_bytes == 24 else 0):
            temp = data[-4:]
            data += xor(temp, data[-key_size_bytes: 4 - key_size_bytes])
    data = data[:expanded_key_size_bytes]

    return data


def aes_encrypt(data, expanded_key):
    """
    Encrypt one block with aes

    @param {int[]} data          16-Byte state
    @param {int[]} expanded_key  176/208/240-Byte expanded key
    @returns {int[]}             16-Byte cipher
    """
    rounds = len(expanded_key) // BLOCK_SIZE_BYTES - 1

    data = xor(data, expanded_key[:BLOCK_SIZE_BYTES])
    for i in range(1, rounds + 1):
        data = sub_bytes(data)
        data = shift_rows(data)
        if i != rounds:
            data = mix_columns(data)
        data = xor(data, expanded_key[i * BLOCK_SIZE_BYTES: (i + 1) * BLOCK_SIZE_BYTES])

    return data


def aes_decrypt(data, expanded_key):
    """
    Decrypt one block with aes

    @param {int[]} data          16-Byte cipher
    @param {int[]} expanded_key  176/208/240-Byte expanded key
    @returns {int[]}             16-Byte state
    """
    rounds = len(expanded_key) // BLOCK_SIZE_BYTES - 1

    for i in range(rounds, 0, -1):
        data = xor(data, expanded_key[i * BLOCK_SIZE_BYTES: (i + 1) * BLOCK_SIZE_BYTES])
        if i != rounds:
            data = mix_columns_inv(data)
        data = shift_rows_inv(data)
        data = sub_bytes_inv(data)
    data = xor(data, expanded_key[:BLOCK_SIZE_BYTES])

    return data


def aes_decrypt_text(data, password, key_size_bytes):
    """
    Decrypt text
    - The first 8 Bytes of decoded 'data' are the 8 high Bytes of the counter
    - The cipher key is retrieved by encrypting the first 16 Byte of 'password'
      with the first 'key_size_bytes' Bytes from 'password' (if necessary filled with 0's)
    - Mode of operation is 'counter'

    @param {str} data                    Base64 encoded string
    @param {str,unicode} password        Password (will be encoded with utf-8)
    @param {int} key_size_bytes          Possible values: 16 for 128-Bit, 24 for 192-Bit or 32 for 256-Bit
    @returns {str}                       Decrypted data
    """
    NONCE_LENGTH_BYTES = 8

    data = bytes_to_intlist(compat_b64decode(data))
    password = bytes_to_intlist(password.encode('utf-8'))

    key = password[:key_size_bytes] + [0] * (key_size_bytes - len(password))
    key = aes_encrypt(key[:BLOCK_SIZE_BYTES], key_expansion(key)) * (key_size_bytes // BLOCK_SIZE_BYTES)

    nonce = data[:NONCE_LENGTH_BYTES]
    cipher = data[NONCE_LENGTH_BYTES:]

    class Counter(object):
        __value = nonce + [0] * (BLOCK_SIZE_BYTES - NONCE_LENGTH_BYTES)

        def next_value(self):
            temp = self.__value
            self.__value = inc(self.__value)
            return temp

    decrypted_data = aes_ctr_decrypt(cipher, key, Counter())
    plaintext = intlist_to_bytes(decrypted_data)

    return plaintext


RCON = (0x8d, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x1b, 0x36)
SBOX = (0x63, 0x7C, 0x77, 0x7B, 0xF2, 0x6B, 0x6F, 0xC5, 0x30, 0x01, 0x67, 0x2B, 0xFE, 0xD7, 0xAB, 0x76,
        0xCA, 0x82, 0xC9, 0x7D, 0xFA, 0x59, 0x47, 0xF0, 0xAD, 0xD4, 0xA2, 0xAF, 0x9C, 0xA4, 0x72, 0xC0,
        0xB7, 0xFD, 0x93, 0x26, 0x36, 0x3F, 0xF7, 0xCC, 0x34, 0xA5, 0xE5, 0xF1, 0x71, 0xD8, 0x31, 0x15,
        0x04, 0xC7, 0x23, 0xC3, 0x18, 0x96, 0x05, 0x9A, 0x07, 0x12, 0x80, 0xE2, 0xEB, 0x27, 0xB2, 0x75,
        0x09, 0x83, 0x2C, 0x1A, 0x1B, 0x6E, 0x5A, 0xA0, 0x52, 0x3B, 0xD6, 0xB3, 0x29, 0xE3, 0x2F, 0x84,
        0x53, 0xD1, 0x00, 0xED, 0x20, 0xFC, 0xB1, 0x5B, 0x6A, 0xCB, 0xBE, 0x39, 0x4A, 0x4C, 0x58, 0xCF,
        0xD0, 0xEF, 0xAA, 0xFB, 0x43, 0x4D, 0x33, 0x85, 0x45, 0xF9, 0x02, 0x7F, 0x50, 0x3C, 0x9F, 0xA8,
        0x51, 0xA3, 0x40, 0x8F, 0x92, 0x9D, 0x38, 0xF5, 0xBC, 0xB6, 0xDA, 0x21, 0x10, 0xFF, 0xF3, 0xD2,
        0xCD, 0x0C, 0x13, 0xEC, 0x5F, 0x97, 0x44, 0x17, 0xC4, 0xA7, 0x7E, 0x3D, 0x64, 0x5D, 0x19, 0x73,
        0x60, 0x81, 0x4F, 0xDC, 0x22, 0x2A, 0x90, 0x88, 0x46, 0xEE, 0xB8, 0x14, 0xDE, 0x5E, 0x0B, 0xDB,
        0xE0, 0x32, 0x3A, 0x0A, 0x49, 0x06, 0x24, 0x5C, 0xC2, 0xD3, 0xAC, 0x62, 0x91, 0x95, 0xE4, 0x79,
        0xE7, 0xC8, 0x37, 0x6D, 0x8D, 0xD5, 0x4E, 0xA9, 0x6C, 0x56, 0xF4, 0xEA, 0x65, 0x7A, 0xAE, 0x08,
        0xBA, 0x78, 0x25, 0x2E, 0x1C, 0xA6, 0xB4, 0xC6, 0xE8, 0xDD, 0x74, 0x1F, 0x4B, 0xBD, 0x8B, 0x8A,
        0x70, 0x3E, 0xB5, 0x66, 0x48, 0x03, 0xF6, 0x0E, 0x61, 0x35, 0x57, 0xB9, 0x86, 0xC1, 0x1D, 0x9E,
        0xE1, 0xF8, 0x98, 0x11, 0x69, 0xD9, 0x8E, 0x94, 0x9B, 0x1E, 0x87, 0xE9, 0xCE, 0x55, 0x28, 0xDF,
        0x8C, 0xA1, 0x89, 0x0D, 0xBF, 0xE6, 0x42, 0x68, 0x41, 0x99, 0x2D, 0x0F, 0xB0, 0x54, 0xBB, 0x16)
SBOX_INV = (0x52, 0x09, 0x6a, 0xd5, 0x30, 0x36, 0xa5, 0x38, 0xbf, 0x40, 0xa3, 0x9e, 0x81, 0xf3, 0xd7, 0xfb,
            0x7c, 0xe3, 0x39, 0x82, 0x9b, 0x2f, 0xff, 0x87, 0x34, 0x8e, 0x43, 0x44, 0xc4, 0xde, 0xe9, 0xcb,
            0x54, 0x7b, 0x94, 0x32, 0xa6, 0xc2, 0x23, 0x3d, 0xee, 0x4c, 0x95, 0x0b, 0x42, 0xfa, 0xc3, 0x4e,
            0x08, 0x2e, 0xa1, 0x66, 0x28, 0xd9, 0x24, 0xb2, 0x76, 0x5b, 0xa2, 0x49, 0x6d, 0x8b, 0xd1, 0x25,
            0x72, 0xf8, 0xf6, 0x64, 0x86, 0x68, 0x98, 0x16, 0xd4, 0xa4, 0x5c, 0xcc, 0x5d, 0x65, 0xb6, 0x92,
            0x6c, 0x70, 0x48, 0x50, 0xfd, 0xed, 0xb9, 0xda, 0x5e, 0x15, 0x46, 0x57, 0xa7, 0x8d, 0x9d, 0x84,
            0x90, 0xd8, 0xab, 0x00, 0x8c, 0xbc, 0xd3, 0x0a, 0xf7, 0xe4, 0x58, 0x05, 0xb8, 0xb3, 0x45, 0x06,
            0xd0, 0x2c, 0x1e, 0x8f, 0xca, 0x3f, 0x0f, 0x02, 0xc1, 0xaf, 0xbd, 0x03, 0x01, 0x13, 0x8a, 0x6b,
            0x3a, 0x91, 0x11, 0x41, 0x4f, 0x67, 0xdc, 0xea, 0x97, 0xf2, 0xcf, 0xce, 0xf0, 0xb4, 0xe6, 0x73,
            0x96, 0xac, 0x74, 0x22, 0xe7, 0xad, 0x35, 0x85, 0xe2, 0xf9, 0x37, 0xe8, 0x1c, 0x75, 0xdf, 0x6e,
            0x47, 0xf1, 0x1a, 0x71, 0x1d, 0x29, 0xc5, 0x89, 0x6f, 0xb7, 0x62, 0x0e, 0xaa, 0x18, 0xbe, 0x1b,
            0xfc, 0x56, 0x3e, 0x4b, 0xc6, 0xd2, 0x79, 0x20, 0x9a, 0xdb, 0xc0, 0xfe, 0x78, 0xcd, 0x5a, 0xf4,
            0x1f, 0xdd, 0xa8, 0x33, 0x88, 0x07, 0xc7, 0x31, 0xb1, 0x12, 0x10, 0x59, 0x27, 0x80, 0xec, 0x5f,
            0x60, 0x51, 0x7f, 0xa9, 0x19, 0xb5, 0x4a, 0x0d, 0x2d, 0xe5, 0x7a, 0x9f, 0x93, 0xc9, 0x9c, 0xef,
            0xa0, 0xe0, 0x3b, 0x4d, 0xae, 0x2a, 0xf5, 0xb0, 0xc8, 0xeb, 0xbb, 0x3c, 0x83, 0x53, 0x99, 0x61,
            0x17, 0x2b, 0x04, 0x7e, 0xba, 0x77, 0xd6, 0x26, 0xe1, 0x69, 0x14, 0x63, 0x55, 0x21, 0x0c, 0x7d)
MIX_COLUMN_MATRIX = ((0x2, 0x3, 0x1, 0x1),
                     (0x1, 0x2, 0x3, 0x1),
                     (0x1, 0x1, 0x2, 0x3),
                     (0x3, 0x1, 0x1, 0x2))
MIX_COLUMN_MATRIX_INV = ((0xE, 0xB, 0xD, 0x9),
                         (0x9, 0xE, 0xB, 0xD),
                         (0xD, 0x9, 0xE, 0xB),
                         (0xB, 0xD, 0x9, 0xE))
RIJNDAEL_EXP_TABLE = (0x01, 0x03, 0x05, 0x0F, 0x11, 0x33, 0x55, 0xFF, 0x1A, 0x2E, 0x72, 0x96, 0xA1, 0xF8, 0x13, 0x35,
                      0x5F, 0xE1, 0x38, 0x48, 0xD8, 0x73, 0x95, 0xA4, 0xF7, 0x02, 0x06, 0x0A, 0x1E, 0x22, 0x66, 0xAA,
                      0xE5, 0x34, 0x5C, 0xE4, 0x37, 0x59, 0xEB, 0x26, 0x6A, 0xBE, 0xD9, 0x70, 0x90, 0xAB, 0xE6, 0x31,
                      0x53, 0xF5, 0x04, 0x0C, 0x14, 0x3C, 0x44, 0xCC, 0x4F, 0xD1, 0x68, 0xB8, 0xD3, 0x6E, 0xB2, 0xCD,
                      0x4C, 0xD4, 0x67, 0xA9, 0xE0, 0x3B, 0x4D, 0xD7, 0x62, 0xA6, 0xF1, 0x08, 0x18, 0x28, 0x78, 0x88,
                      0x83, 0x9E, 0xB9, 0xD0, 0x6B, 0xBD, 0xDC, 0x7F, 0x81, 0x98, 0xB3, 0xCE, 0x49, 0xDB, 0x76, 0x9A,
                      0xB5, 0xC4, 0x57, 0xF9, 0x10, 0x30, 0x50, 0xF0, 0x0B, 0x1D, 0x27, 0x69, 0xBB, 0xD6, 0x61, 0xA3,
                      0xFE, 0x19, 0x2B, 0x7D, 0x87, 0x92, 0xAD, 0xEC, 0x2F, 0x71, 0x93, 0xAE, 0xE9, 0x20, 0x60, 0xA0,
                      0xFB, 0x16, 0x3A, 0x4E, 0xD2, 0x6D, 0xB7, 0xC2, 0x5D, 0xE7, 0x32, 0x56, 0xFA, 0x15, 0x3F, 0x41,
                      0xC3, 0x5E, 0xE2, 0x3D, 0x47, 0xC9, 0x40, 0xC0, 0x5B, 0xED, 0x2C, 0x74, 0x9C, 0xBF, 0xDA, 0x75,
                      0x9F, 0xBA, 0xD5, 0x64, 0xAC, 0xEF, 0x2A, 0x7E, 0x82, 0x9D, 0xBC, 0xDF, 0x7A, 0x8E, 0x89, 0x80,
                      0x9B, 0xB6, 0xC1, 0x58, 0xE8, 0x23, 0x65, 0xAF, 0xEA, 0x25, 0x6F, 0xB1, 0xC8, 0x43, 0xC5, 0x54,
                      0xFC, 0x1F, 0x21, 0x63, 0xA5, 0xF4, 0x07, 0x09, 0x1B, 0x2D, 0x77, 0x99, 0xB0, 0xCB, 0x46, 0xCA,
                      0x45, 0xCF, 0x4A, 0xDE, 0x79, 0x8B, 0x86, 0x91, 0xA8, 0xE3, 0x3E, 0x42, 0xC6, 0x51, 0xF3, 0x0E,
                      0x12, 0x36, 0x5A, 0xEE, 0x29, 0x7B, 0x8D, 0x8C, 0x8F, 0x8A, 0x85, 0x94, 0xA7, 0xF2, 0x0D, 0x17,
                      0x39, 0x4B, 0xDD, 0x7C, 0x84, 0x97, 0xA2, 0xFD, 0x1C, 0x24, 0x6C, 0xB4, 0xC7, 0x52, 0xF6, 0x01)
RIJNDAEL_LOG_TABLE = (0x00, 0x00, 0x19, 0x01, 0x32, 0x02, 0x1a, 0xc6, 0x4b, 0xc7, 0x1b, 0x68, 0x33, 0xee, 0xdf, 0x03,
                      0x64, 0x04, 0xe0, 0x0e, 0x34, 0x8d, 0x81, 0xef, 0x4c, 0x71, 0x08, 0xc8, 0xf8, 0x69, 0x1c, 0xc1,
                      0x7d, 0xc2, 0x1d, 0xb5, 0xf9, 0xb9, 0x27, 0x6a, 0x4d, 0xe4, 0xa6, 0x72, 0x9a, 0xc9, 0x09, 0x78,
                      0x65, 0x2f, 0x8a, 0x05, 0x21, 0x0f, 0xe1, 0x24, 0x12, 0xf0, 0x82, 0x45, 0x35, 0x93, 0xda, 0x8e,
                      0x96, 0x8f, 0xdb, 0xbd, 0x36, 0xd0, 0xce, 0x94, 0x13, 0x5c, 0xd2, 0xf1, 0x40, 0x46, 0x83, 0x38,
                      0x66, 0xdd, 0xfd, 0x30, 0xbf, 0x06, 0x8b, 0x62, 0xb3, 0x25, 0xe2, 0x98, 0x22, 0x88, 0x91, 0x10,
                      0x7e, 0x6e, 0x48, 0xc3, 0xa3, 0xb6, 0x1e, 0x42, 0x3a, 0x6b, 0x28, 0x54, 0xfa, 0x85, 0x3d, 0xba,
                      0x2b, 0x79, 0x0a, 0x15, 0x9b, 0x9f, 0x5e, 0xca, 0x4e, 0xd4, 0xac, 0xe5, 0xf3, 0x73, 0xa7, 0x57,
                      0xaf, 0x58, 0xa8, 0x50, 0xf4, 0xea, 0xd6, 0x74, 0x4f, 0xae, 0xe9, 0xd5, 0xe7, 0xe6, 0xad, 0xe8,
                      0x2c, 0xd7, 0x75, 0x7a, 0xeb, 0x16, 0x0b, 0xf5, 0x59, 0xcb, 0x5f, 0xb0, 0x9c, 0xa9, 0x51, 0xa0,
                      0x7f, 0x0c, 0xf6, 0x6f, 0x17, 0xc4, 0x49, 0xec, 0xd8, 0x43, 0x1f, 0x2d, 0xa4, 0x76, 0x7b, 0xb7,
                      0xcc, 0xbb, 0x3e, 0x5a, 0xfb, 0x60, 0xb1, 0x86, 0x3b, 0x52, 0xa1, 0x6c, 0xaa, 0x55, 0x29, 0x9d,
                      0x97, 0xb2, 0x87, 0x90, 0x61, 0xbe, 0xdc, 0xfc, 0xbc, 0x95, 0xcf, 0xcd, 0x37, 0x3f, 0x5b, 0xd1,
                      0x53, 0x39, 0x84, 0x3c, 0x41, 0xa2, 0x6d, 0x47, 0x14, 0x2a, 0x9e, 0x5d, 0x56, 0xf2, 0xd3, 0xab,
                      0x44, 0x11, 0x92, 0xd9, 0x23, 0x20, 0x2e, 0x89, 0xb4, 0x7c, 0xb8, 0x26, 0x77, 0x99, 0xe3, 0xa5,
                      0x67, 0x4a, 0xed, 0xde, 0xc5, 0x31, 0xfe, 0x18, 0x0d, 0x63, 0x8c, 0x80, 0xc0, 0xf7, 0x70, 0x07)


def sub_bytes(data):
    return [SBOX[x] for x in data]


def sub_bytes_inv(data):
    return [SBOX_INV[x] for x in data]


def rotate(data):
    return data[1:] + [data[0]]


def key_schedule_core(data, rcon_iteration):
    data = rotate(data)
    data = sub_bytes(data)
    data[0] = data[0] ^ RCON[rcon_iteration]

    return data


def xor(data1, data2):
    return [x ^ y for x, y in zip(data1, data2)]


def rijndael_mul(a, b):
    if(a == 0 or b == 0):
        return 0
    return RIJNDAEL_EXP_TABLE[(RIJNDAEL_LOG_TABLE[a] + RIJNDAEL_LOG_TABLE[b]) % 0xFF]


def mix_column(data, matrix):
    data_mixed = []
    for row in range(4):
        mixed = 0
        for column in range(4):
            # xor is (+) and (-)
            mixed ^= rijndael_mul(data[column], matrix[row][column])
        data_mixed.append(mixed)
    return data_mixed


def mix_columns(data, matrix=MIX_COLUMN_MATRIX):
    data_mixed = []
    for i in range(4):
        column = data[i * 4: (i + 1) * 4]
        data_mixed += mix_column(column, matrix)
    return data_mixed


def mix_columns_inv(data):
    return mix_columns(data, MIX_COLUMN_MATRIX_INV)


def shift_rows(data):
    data_shifted = []
    for column in range(4):
        for row in range(4):
            data_shifted.append(data[((column + row) & 0b11) * 4 + row])
    return data_shifted


def shift_rows_inv(data):
    data_shifted = []
    for column in range(4):
        for row in range(4):
            data_shifted.append(data[((column - row) & 0b11) * 4 + row])
    return data_shifted


def inc(data):
    data = data[:]  # copy
    for i in range(len(data) - 1, -1, -1):
        if data[i] == 255:
            data[i] = 0
        else:
            data[i] = data[i] + 1
            break
    return data


__all__ = ['aes_encrypt', 'key_expansion', 'aes_ctr_decrypt', 'aes_cbc_decrypt', 'aes_decrypt_text']
from __future__ import unicode_literals

import errno
import io
import json
import os
import re
import shutil
import traceback

from .compat import compat_getenv
from .utils import (
    expand_path,
    write_json_file,
)


class Cache(object):
    def __init__(self, ydl):
        self._ydl = ydl

    def _get_root_dir(self):
        res = self._ydl.params.get('cachedir')
        if res is None:
            cache_root = compat_getenv('XDG_CACHE_HOME', '~/.cache')
            res = os.path.join(cache_root, 'youtube-dl')
        return expand_path(res)

    def _get_cache_fn(self, section, key, dtype):
        assert re.match(r'^[a-zA-Z0-9_.-]+$', section), \
            'invalid section %r' % section
        assert re.match(r'^[a-zA-Z0-9_.-]+$', key), 'invalid key %r' % key
        return os.path.join(
            self._get_root_dir(), section, '%s.%s' % (key, dtype))

    @property
    def enabled(self):
        return self._ydl.params.get('cachedir') is not False

    def store(self, section, key, data, dtype='json'):
        assert dtype in ('json',)

        if not self.enabled:
            return

        fn = self._get_cache_fn(section, key, dtype)
        try:
            try:
                os.makedirs(os.path.dirname(fn))
            except OSError as ose:
                if ose.errno != errno.EEXIST:
                    raise
            write_json_file(data, fn)
        except Exception:
            tb = traceback.format_exc()
            self._ydl.report_warning(
                'Writing cache to %r failed: %s' % (fn, tb))

    def load(self, section, key, dtype='json', default=None):
        assert dtype in ('json',)

        if not self.enabled:
            return default

        cache_fn = self._get_cache_fn(section, key, dtype)
        try:
            try:
                with io.open(cache_fn, 'r', encoding='utf-8') as cachef:
                    return json.load(cachef)
            except ValueError:
                try:
                    file_size = os.path.getsize(cache_fn)
                except (OSError, IOError) as oe:
                    file_size = str(oe)
                self._ydl.report_warning(
                    'Cache retrieval from %s failed (%s)' % (cache_fn, file_size))
        except IOError:
            pass  # No cache available

        return default

    def remove(self):
        if not self.enabled:
            self._ydl.to_screen('Cache is disabled (Did you combine --no-cache-dir and --rm-cache-dir?)')
            return

        cachedir = self._get_root_dir()
        if not any((term in cachedir) for term in ('cache', 'tmp')):
            raise Exception('Not removing directory %s - this does not look like a cache dir' % cachedir)

        self._ydl.to_screen(
            'Removing cache dir %s .' % cachedir, skip_eol=True)
        if os.path.exists(cachedir):
            self._ydl.to_screen('.', skip_eol=True)
            shutil.rmtree(cachedir)
        self._ydl.to_screen('.')
# coding: utf-8
from __future__ import unicode_literals

import base64
import binascii
import collections
import ctypes
import email
import getpass
import io
import itertools
import optparse
import os
import platform
import re
import shlex
import shutil
import socket
import struct
import subprocess
import sys
import xml.etree.ElementTree


try:
    import urllib.request as compat_urllib_request
except ImportError:  # Python 2
    import urllib2 as compat_urllib_request

try:
    import urllib.error as compat_urllib_error
except ImportError:  # Python 2
    import urllib2 as compat_urllib_error

try:
    import urllib.parse as compat_urllib_parse
except ImportError:  # Python 2
    import urllib as compat_urllib_parse

try:
    from urllib.parse import urlparse as compat_urllib_parse_urlparse
except ImportError:  # Python 2
    from urlparse import urlparse as compat_urllib_parse_urlparse

try:
    import urllib.parse as compat_urlparse
except ImportError:  # Python 2
    import urlparse as compat_urlparse

try:
    import urllib.response as compat_urllib_response
except ImportError:  # Python 2
    import urllib as compat_urllib_response

try:
    import http.cookiejar as compat_cookiejar
except ImportError:  # Python 2
    import cookielib as compat_cookiejar

if sys.version_info[0] == 2:
    class compat_cookiejar_Cookie(compat_cookiejar.Cookie):
        def __init__(self, version, name, value, *args, **kwargs):
            if isinstance(name, compat_str):
                name = name.encode()
            if isinstance(value, compat_str):
                value = value.encode()
            compat_cookiejar.Cookie.__init__(self, version, name, value, *args, **kwargs)
else:
    compat_cookiejar_Cookie = compat_cookiejar.Cookie

try:
    import http.cookies as compat_cookies
except ImportError:  # Python 2
    import Cookie as compat_cookies

try:
    import html.entities as compat_html_entities
except ImportError:  # Python 2
    import htmlentitydefs as compat_html_entities

try:  # Python >= 3.3
    compat_html_entities_html5 = compat_html_entities.html5
except AttributeError:
    # Copied from CPython 3.5.1 html/entities.py
    compat_html_entities_html5 = {
        'Aacute': '\xc1',
        'aacute': '\xe1',
        'Aacute;': '\xc1',
        'aacute;': '\xe1',
        'Abreve;': '\u0102',
        'abreve;': '\u0103',
        'ac;': '\u223e',
        'acd;': '\u223f',
        'acE;': '\u223e\u0333',
        'Acirc': '\xc2',
        'acirc': '\xe2',
        'Acirc;': '\xc2',
        'acirc;': '\xe2',
        'acute': '\xb4',
        'acute;': '\xb4',
        'Acy;': '\u0410',
        'acy;': '\u0430',
        'AElig': '\xc6',
        'aelig': '\xe6',
        'AElig;': '\xc6',
        'aelig;': '\xe6',
        'af;': '\u2061',
        'Afr;': '\U0001d504',
        'afr;': '\U0001d51e',
        'Agrave': '\xc0',
        'agrave': '\xe0',
        'Agrave;': '\xc0',
        'agrave;': '\xe0',
        'alefsym;': '\u2135',
        'aleph;': '\u2135',
        'Alpha;': '\u0391',
        'alpha;': '\u03b1',
        'Amacr;': '\u0100',
        'amacr;': '\u0101',
        'amalg;': '\u2a3f',
        'AMP': '&',
        'amp': '&',
        'AMP;': '&',
        'amp;': '&',
        'And;': '\u2a53',
        'and;': '\u2227',
        'andand;': '\u2a55',
        'andd;': '\u2a5c',
        'andslope;': '\u2a58',
        'andv;': '\u2a5a',
        'ang;': '\u2220',
        'ange;': '\u29a4',
        'angle;': '\u2220',
        'angmsd;': '\u2221',
        'angmsdaa;': '\u29a8',
        'angmsdab;': '\u29a9',
        'angmsdac;': '\u29aa',
        'angmsdad;': '\u29ab',
        'angmsdae;': '\u29ac',
        'angmsdaf;': '\u29ad',
        'angmsdag;': '\u29ae',
        'angmsdah;': '\u29af',
        'angrt;': '\u221f',
        'angrtvb;': '\u22be',
        'angrtvbd;': '\u299d',
        'angsph;': '\u2222',
        'angst;': '\xc5',
        'angzarr;': '\u237c',
        'Aogon;': '\u0104',
        'aogon;': '\u0105',
        'Aopf;': '\U0001d538',
        'aopf;': '\U0001d552',
        'ap;': '\u2248',
        'apacir;': '\u2a6f',
        'apE;': '\u2a70',
        'ape;': '\u224a',
        'apid;': '\u224b',
        'apos;': "'",
        'ApplyFunction;': '\u2061',
        'approx;': '\u2248',
        'approxeq;': '\u224a',
        'Aring': '\xc5',
        'aring': '\xe5',
        'Aring;': '\xc5',
        'aring;': '\xe5',
        'Ascr;': '\U0001d49c',
        'ascr;': '\U0001d4b6',
        'Assign;': '\u2254',
        'ast;': '*',
        'asymp;': '\u2248',
        'asympeq;': '\u224d',
        'Atilde': '\xc3',
        'atilde': '\xe3',
        'Atilde;': '\xc3',
        'atilde;': '\xe3',
        'Auml': '\xc4',
        'auml': '\xe4',
        'Auml;': '\xc4',
        'auml;': '\xe4',
        'awconint;': '\u2233',
        'awint;': '\u2a11',
        'backcong;': '\u224c',
        'backepsilon;': '\u03f6',
        'backprime;': '\u2035',
        'backsim;': '\u223d',
        'backsimeq;': '\u22cd',
        'Backslash;': '\u2216',
        'Barv;': '\u2ae7',
        'barvee;': '\u22bd',
        'Barwed;': '\u2306',
        'barwed;': '\u2305',
        'barwedge;': '\u2305',
        'bbrk;': '\u23b5',
        'bbrktbrk;': '\u23b6',
        'bcong;': '\u224c',
        'Bcy;': '\u0411',
        'bcy;': '\u0431',
        'bdquo;': '\u201e',
        'becaus;': '\u2235',
        'Because;': '\u2235',
        'because;': '\u2235',
        'bemptyv;': '\u29b0',
        'bepsi;': '\u03f6',
        'bernou;': '\u212c',
        'Bernoullis;': '\u212c',
        'Beta;': '\u0392',
        'beta;': '\u03b2',
        'beth;': '\u2136',
        'between;': '\u226c',
        'Bfr;': '\U0001d505',
        'bfr;': '\U0001d51f',
        'bigcap;': '\u22c2',
        'bigcirc;': '\u25ef',
        'bigcup;': '\u22c3',
        'bigodot;': '\u2a00',
        'bigoplus;': '\u2a01',
        'bigotimes;': '\u2a02',
        'bigsqcup;': '\u2a06',
        'bigstar;': '\u2605',
        'bigtriangledown;': '\u25bd',
        'bigtriangleup;': '\u25b3',
        'biguplus;': '\u2a04',
        'bigvee;': '\u22c1',
        'bigwedge;': '\u22c0',
        'bkarow;': '\u290d',
        'blacklozenge;': '\u29eb',
        'blacksquare;': '\u25aa',
        'blacktriangle;': '\u25b4',
        'blacktriangledown;': '\u25be',
        'blacktriangleleft;': '\u25c2',
        'blacktriangleright;': '\u25b8',
        'blank;': '\u2423',
        'blk12;': '\u2592',
        'blk14;': '\u2591',
        'blk34;': '\u2593',
        'block;': '\u2588',
        'bne;': '=\u20e5',
        'bnequiv;': '\u2261\u20e5',
        'bNot;': '\u2aed',
        'bnot;': '\u2310',
        'Bopf;': '\U0001d539',
        'bopf;': '\U0001d553',
        'bot;': '\u22a5',
        'bottom;': '\u22a5',
        'bowtie;': '\u22c8',
        'boxbox;': '\u29c9',
        'boxDL;': '\u2557',
        'boxDl;': '\u2556',
        'boxdL;': '\u2555',
        'boxdl;': '\u2510',
        'boxDR;': '\u2554',
        'boxDr;': '\u2553',
        'boxdR;': '\u2552',
        'boxdr;': '\u250c',
        'boxH;': '\u2550',
        'boxh;': '\u2500',
        'boxHD;': '\u2566',
        'boxHd;': '\u2564',
        'boxhD;': '\u2565',
        'boxhd;': '\u252c',
        'boxHU;': '\u2569',
        'boxHu;': '\u2567',
        'boxhU;': '\u2568',
        'boxhu;': '\u2534',
        'boxminus;': '\u229f',
        'boxplus;': '\u229e',
        'boxtimes;': '\u22a0',
        'boxUL;': '\u255d',
        'boxUl;': '\u255c',
        'boxuL;': '\u255b',
        'boxul;': '\u2518',
        'boxUR;': '\u255a',
        'boxUr;': '\u2559',
        'boxuR;': '\u2558',
        'boxur;': '\u2514',
        'boxV;': '\u2551',
        'boxv;': '\u2502',
        'boxVH;': '\u256c',
        'boxVh;': '\u256b',
        'boxvH;': '\u256a',
        'boxvh;': '\u253c',
        'boxVL;': '\u2563',
        'boxVl;': '\u2562',
        'boxvL;': '\u2561',
        'boxvl;': '\u2524',
        'boxVR;': '\u2560',
        'boxVr;': '\u255f',
        'boxvR;': '\u255e',
        'boxvr;': '\u251c',
        'bprime;': '\u2035',
        'Breve;': '\u02d8',
        'breve;': '\u02d8',
        'brvbar': '\xa6',
        'brvbar;': '\xa6',
        'Bscr;': '\u212c',
        'bscr;': '\U0001d4b7',
        'bsemi;': '\u204f',
        'bsim;': '\u223d',
        'bsime;': '\u22cd',
        'bsol;': '\\',
        'bsolb;': '\u29c5',
        'bsolhsub;': '\u27c8',
        'bull;': '\u2022',
        'bullet;': '\u2022',
        'bump;': '\u224e',
        'bumpE;': '\u2aae',
        'bumpe;': '\u224f',
        'Bumpeq;': '\u224e',
        'bumpeq;': '\u224f',
        'Cacute;': '\u0106',
        'cacute;': '\u0107',
        'Cap;': '\u22d2',
        'cap;': '\u2229',
        'capand;': '\u2a44',
        'capbrcup;': '\u2a49',
        'capcap;': '\u2a4b',
        'capcup;': '\u2a47',
        'capdot;': '\u2a40',
        'CapitalDifferentialD;': '\u2145',
        'caps;': '\u2229\ufe00',
        'caret;': '\u2041',
        'caron;': '\u02c7',
        'Cayleys;': '\u212d',
        'ccaps;': '\u2a4d',
        'Ccaron;': '\u010c',
        'ccaron;': '\u010d',
        'Ccedil': '\xc7',
        'ccedil': '\xe7',
        'Ccedil;': '\xc7',
        'ccedil;': '\xe7',
        'Ccirc;': '\u0108',
        'ccirc;': '\u0109',
        'Cconint;': '\u2230',
        'ccups;': '\u2a4c',
        'ccupssm;': '\u2a50',
        'Cdot;': '\u010a',
        'cdot;': '\u010b',
        'cedil': '\xb8',
        'cedil;': '\xb8',
        'Cedilla;': '\xb8',
        'cemptyv;': '\u29b2',
        'cent': '\xa2',
        'cent;': '\xa2',
        'CenterDot;': '\xb7',
        'centerdot;': '\xb7',
        'Cfr;': '\u212d',
        'cfr;': '\U0001d520',
        'CHcy;': '\u0427',
        'chcy;': '\u0447',
        'check;': '\u2713',
        'checkmark;': '\u2713',
        'Chi;': '\u03a7',
        'chi;': '\u03c7',
        'cir;': '\u25cb',
        'circ;': '\u02c6',
        'circeq;': '\u2257',
        'circlearrowleft;': '\u21ba',
        'circlearrowright;': '\u21bb',
        'circledast;': '\u229b',
        'circledcirc;': '\u229a',
        'circleddash;': '\u229d',
        'CircleDot;': '\u2299',
        'circledR;': '\xae',
        'circledS;': '\u24c8',
        'CircleMinus;': '\u2296',
        'CirclePlus;': '\u2295',
        'CircleTimes;': '\u2297',
        'cirE;': '\u29c3',
        'cire;': '\u2257',
        'cirfnint;': '\u2a10',
        'cirmid;': '\u2aef',
        'cirscir;': '\u29c2',
        'ClockwiseContourIntegral;': '\u2232',
        'CloseCurlyDoubleQuote;': '\u201d',
        'CloseCurlyQuote;': '\u2019',
        'clubs;': '\u2663',
        'clubsuit;': '\u2663',
        'Colon;': '\u2237',
        'colon;': ':',
        'Colone;': '\u2a74',
        'colone;': '\u2254',
        'coloneq;': '\u2254',
        'comma;': ',',
        'commat;': '@',
        'comp;': '\u2201',
        'compfn;': '\u2218',
        'complement;': '\u2201',
        'complexes;': '\u2102',
        'cong;': '\u2245',
        'congdot;': '\u2a6d',
        'Congruent;': '\u2261',
        'Conint;': '\u222f',
        'conint;': '\u222e',
        'ContourIntegral;': '\u222e',
        'Copf;': '\u2102',
        'copf;': '\U0001d554',
        'coprod;': '\u2210',
        'Coproduct;': '\u2210',
        'COPY': '\xa9',
        'copy': '\xa9',
        'COPY;': '\xa9',
        'copy;': '\xa9',
        'copysr;': '\u2117',
        'CounterClockwiseContourIntegral;': '\u2233',
        'crarr;': '\u21b5',
        'Cross;': '\u2a2f',
        'cross;': '\u2717',
        'Cscr;': '\U0001d49e',
        'cscr;': '\U0001d4b8',
        'csub;': '\u2acf',
        'csube;': '\u2ad1',
        'csup;': '\u2ad0',
        'csupe;': '\u2ad2',
        'ctdot;': '\u22ef',
        'cudarrl;': '\u2938',
        'cudarrr;': '\u2935',
        'cuepr;': '\u22de',
        'cuesc;': '\u22df',
        'cularr;': '\u21b6',
        'cularrp;': '\u293d',
        'Cup;': '\u22d3',
        'cup;': '\u222a',
        'cupbrcap;': '\u2a48',
        'CupCap;': '\u224d',
        'cupcap;': '\u2a46',
        'cupcup;': '\u2a4a',
        'cupdot;': '\u228d',
        'cupor;': '\u2a45',
        'cups;': '\u222a\ufe00',
        'curarr;': '\u21b7',
        'curarrm;': '\u293c',
        'curlyeqprec;': '\u22de',
        'curlyeqsucc;': '\u22df',
        'curlyvee;': '\u22ce',
        'curlywedge;': '\u22cf',
        'curren': '\xa4',
        'curren;': '\xa4',
        'curvearrowleft;': '\u21b6',
        'curvearrowright;': '\u21b7',
        'cuvee;': '\u22ce',
        'cuwed;': '\u22cf',
        'cwconint;': '\u2232',
        'cwint;': '\u2231',
        'cylcty;': '\u232d',
        'Dagger;': '\u2021',
        'dagger;': '\u2020',
        'daleth;': '\u2138',
        'Darr;': '\u21a1',
        'dArr;': '\u21d3',
        'darr;': '\u2193',
        'dash;': '\u2010',
        'Dashv;': '\u2ae4',
        'dashv;': '\u22a3',
        'dbkarow;': '\u290f',
        'dblac;': '\u02dd',
        'Dcaron;': '\u010e',
        'dcaron;': '\u010f',
        'Dcy;': '\u0414',
        'dcy;': '\u0434',
        'DD;': '\u2145',
        'dd;': '\u2146',
        'ddagger;': '\u2021',
        'ddarr;': '\u21ca',
        'DDotrahd;': '\u2911',
        'ddotseq;': '\u2a77',
        'deg': '\xb0',
        'deg;': '\xb0',
        'Del;': '\u2207',
        'Delta;': '\u0394',
        'delta;': '\u03b4',
        'demptyv;': '\u29b1',
        'dfisht;': '\u297f',
        'Dfr;': '\U0001d507',
        'dfr;': '\U0001d521',
        'dHar;': '\u2965',
        'dharl;': '\u21c3',
        'dharr;': '\u21c2',
        'DiacriticalAcute;': '\xb4',
        'DiacriticalDot;': '\u02d9',
        'DiacriticalDoubleAcute;': '\u02dd',
        'DiacriticalGrave;': '`',
        'DiacriticalTilde;': '\u02dc',
        'diam;': '\u22c4',
        'Diamond;': '\u22c4',
        'diamond;': '\u22c4',
        'diamondsuit;': '\u2666',
        'diams;': '\u2666',
        'die;': '\xa8',
        'DifferentialD;': '\u2146',
        'digamma;': '\u03dd',
        'disin;': '\u22f2',
        'div;': '\xf7',
        'divide': '\xf7',
        'divide;': '\xf7',
        'divideontimes;': '\u22c7',
        'divonx;': '\u22c7',
        'DJcy;': '\u0402',
        'djcy;': '\u0452',
        'dlcorn;': '\u231e',
        'dlcrop;': '\u230d',
        'dollar;': '$',
        'Dopf;': '\U0001d53b',
        'dopf;': '\U0001d555',
        'Dot;': '\xa8',
        'dot;': '\u02d9',
        'DotDot;': '\u20dc',
        'doteq;': '\u2250',
        'doteqdot;': '\u2251',
        'DotEqual;': '\u2250',
        'dotminus;': '\u2238',
        'dotplus;': '\u2214',
        'dotsquare;': '\u22a1',
        'doublebarwedge;': '\u2306',
        'DoubleContourIntegral;': '\u222f',
        'DoubleDot;': '\xa8',
        'DoubleDownArrow;': '\u21d3',
        'DoubleLeftArrow;': '\u21d0',
        'DoubleLeftRightArrow;': '\u21d4',
        'DoubleLeftTee;': '\u2ae4',
        'DoubleLongLeftArrow;': '\u27f8',
        'DoubleLongLeftRightArrow;': '\u27fa',
        'DoubleLongRightArrow;': '\u27f9',
        'DoubleRightArrow;': '\u21d2',
        'DoubleRightTee;': '\u22a8',
        'DoubleUpArrow;': '\u21d1',
        'DoubleUpDownArrow;': '\u21d5',
        'DoubleVerticalBar;': '\u2225',
        'DownArrow;': '\u2193',
        'Downarrow;': '\u21d3',
        'downarrow;': '\u2193',
        'DownArrowBar;': '\u2913',
        'DownArrowUpArrow;': '\u21f5',
        'DownBreve;': '\u0311',
        'downdownarrows;': '\u21ca',
        'downharpoonleft;': '\u21c3',
        'downharpoonright;': '\u21c2',
        'DownLeftRightVector;': '\u2950',
        'DownLeftTeeVector;': '\u295e',
        'DownLeftVector;': '\u21bd',
        'DownLeftVectorBar;': '\u2956',
        'DownRightTeeVector;': '\u295f',
        'DownRightVector;': '\u21c1',
        'DownRightVectorBar;': '\u2957',
        'DownTee;': '\u22a4',
        'DownTeeArrow;': '\u21a7',
        'drbkarow;': '\u2910',
        'drcorn;': '\u231f',
        'drcrop;': '\u230c',
        'Dscr;': '\U0001d49f',
        'dscr;': '\U0001d4b9',
        'DScy;': '\u0405',
        'dscy;': '\u0455',
        'dsol;': '\u29f6',
        'Dstrok;': '\u0110',
        'dstrok;': '\u0111',
        'dtdot;': '\u22f1',
        'dtri;': '\u25bf',
        'dtrif;': '\u25be',
        'duarr;': '\u21f5',
        'duhar;': '\u296f',
        'dwangle;': '\u29a6',
        'DZcy;': '\u040f',
        'dzcy;': '\u045f',
        'dzigrarr;': '\u27ff',
        'Eacute': '\xc9',
        'eacute': '\xe9',
        'Eacute;': '\xc9',
        'eacute;': '\xe9',
        'easter;': '\u2a6e',
        'Ecaron;': '\u011a',
        'ecaron;': '\u011b',
        'ecir;': '\u2256',
        'Ecirc': '\xca',
        'ecirc': '\xea',
        'Ecirc;': '\xca',
        'ecirc;': '\xea',
        'ecolon;': '\u2255',
        'Ecy;': '\u042d',
        'ecy;': '\u044d',
        'eDDot;': '\u2a77',
        'Edot;': '\u0116',
        'eDot;': '\u2251',
        'edot;': '\u0117',
        'ee;': '\u2147',
        'efDot;': '\u2252',
        'Efr;': '\U0001d508',
        'efr;': '\U0001d522',
        'eg;': '\u2a9a',
        'Egrave': '\xc8',
        'egrave': '\xe8',
        'Egrave;': '\xc8',
        'egrave;': '\xe8',
        'egs;': '\u2a96',
        'egsdot;': '\u2a98',
        'el;': '\u2a99',
        'Element;': '\u2208',
        'elinters;': '\u23e7',
        'ell;': '\u2113',
        'els;': '\u2a95',
        'elsdot;': '\u2a97',
        'Emacr;': '\u0112',
        'emacr;': '\u0113',
        'empty;': '\u2205',
        'emptyset;': '\u2205',
        'EmptySmallSquare;': '\u25fb',
        'emptyv;': '\u2205',
        'EmptyVerySmallSquare;': '\u25ab',
        'emsp13;': '\u2004',
        'emsp14;': '\u2005',
        'emsp;': '\u2003',
        'ENG;': '\u014a',
        'eng;': '\u014b',
        'ensp;': '\u2002',
        'Eogon;': '\u0118',
        'eogon;': '\u0119',
        'Eopf;': '\U0001d53c',
        'eopf;': '\U0001d556',
        'epar;': '\u22d5',
        'eparsl;': '\u29e3',
        'eplus;': '\u2a71',
        'epsi;': '\u03b5',
        'Epsilon;': '\u0395',
        'epsilon;': '\u03b5',
        'epsiv;': '\u03f5',
        'eqcirc;': '\u2256',
        'eqcolon;': '\u2255',
        'eqsim;': '\u2242',
        'eqslantgtr;': '\u2a96',
        'eqslantless;': '\u2a95',
        'Equal;': '\u2a75',
        'equals;': '=',
        'EqualTilde;': '\u2242',
        'equest;': '\u225f',
        'Equilibrium;': '\u21cc',
        'equiv;': '\u2261',
        'equivDD;': '\u2a78',
        'eqvparsl;': '\u29e5',
        'erarr;': '\u2971',
        'erDot;': '\u2253',
        'Escr;': '\u2130',
        'escr;': '\u212f',
        'esdot;': '\u2250',
        'Esim;': '\u2a73',
        'esim;': '\u2242',
        'Eta;': '\u0397',
        'eta;': '\u03b7',
        'ETH': '\xd0',
        'eth': '\xf0',
        'ETH;': '\xd0',
        'eth;': '\xf0',
        'Euml': '\xcb',
        'euml': '\xeb',
        'Euml;': '\xcb',
        'euml;': '\xeb',
        'euro;': '\u20ac',
        'excl;': '!',
        'exist;': '\u2203',
        'Exists;': '\u2203',
        'expectation;': '\u2130',
        'ExponentialE;': '\u2147',
        'exponentiale;': '\u2147',
        'fallingdotseq;': '\u2252',
        'Fcy;': '\u0424',
        'fcy;': '\u0444',
        'female;': '\u2640',
        'ffilig;': '\ufb03',
        'fflig;': '\ufb00',
        'ffllig;': '\ufb04',
        'Ffr;': '\U0001d509',
        'ffr;': '\U0001d523',
        'filig;': '\ufb01',
        'FilledSmallSquare;': '\u25fc',
        'FilledVerySmallSquare;': '\u25aa',
        'fjlig;': 'fj',
        'flat;': '\u266d',
        'fllig;': '\ufb02',
        'fltns;': '\u25b1',
        'fnof;': '\u0192',
        'Fopf;': '\U0001d53d',
        'fopf;': '\U0001d557',
        'ForAll;': '\u2200',
        'forall;': '\u2200',
        'fork;': '\u22d4',
        'forkv;': '\u2ad9',
        'Fouriertrf;': '\u2131',
        'fpartint;': '\u2a0d',
        'frac12': '\xbd',
        'frac12;': '\xbd',
        'frac13;': '\u2153',
        'frac14': '\xbc',
        'frac14;': '\xbc',
        'frac15;': '\u2155',
        'frac16;': '\u2159',
        'frac18;': '\u215b',
        'frac23;': '\u2154',
        'frac25;': '\u2156',
        'frac34': '\xbe',
        'frac34;': '\xbe',
        'frac35;': '\u2157',
        'frac38;': '\u215c',
        'frac45;': '\u2158',
        'frac56;': '\u215a',
        'frac58;': '\u215d',
        'frac78;': '\u215e',
        'frasl;': '\u2044',
        'frown;': '\u2322',
        'Fscr;': '\u2131',
        'fscr;': '\U0001d4bb',
        'gacute;': '\u01f5',
        'Gamma;': '\u0393',
        'gamma;': '\u03b3',
        'Gammad;': '\u03dc',
        'gammad;': '\u03dd',
        'gap;': '\u2a86',
        'Gbreve;': '\u011e',
        'gbreve;': '\u011f',
        'Gcedil;': '\u0122',
        'Gcirc;': '\u011c',
        'gcirc;': '\u011d',
        'Gcy;': '\u0413',
        'gcy;': '\u0433',
        'Gdot;': '\u0120',
        'gdot;': '\u0121',
        'gE;': '\u2267',
        'ge;': '\u2265',
        'gEl;': '\u2a8c',
        'gel;': '\u22db',
        'geq;': '\u2265',
        'geqq;': '\u2267',
        'geqslant;': '\u2a7e',
        'ges;': '\u2a7e',
        'gescc;': '\u2aa9',
        'gesdot;': '\u2a80',
        'gesdoto;': '\u2a82',
        'gesdotol;': '\u2a84',
        'gesl;': '\u22db\ufe00',
        'gesles;': '\u2a94',
        'Gfr;': '\U0001d50a',
        'gfr;': '\U0001d524',
        'Gg;': '\u22d9',
        'gg;': '\u226b',
        'ggg;': '\u22d9',
        'gimel;': '\u2137',
        'GJcy;': '\u0403',
        'gjcy;': '\u0453',
        'gl;': '\u2277',
        'gla;': '\u2aa5',
        'glE;': '\u2a92',
        'glj;': '\u2aa4',
        'gnap;': '\u2a8a',
        'gnapprox;': '\u2a8a',
        'gnE;': '\u2269',
        'gne;': '\u2a88',
        'gneq;': '\u2a88',
        'gneqq;': '\u2269',
        'gnsim;': '\u22e7',
        'Gopf;': '\U0001d53e',
        'gopf;': '\U0001d558',
        'grave;': '`',
        'GreaterEqual;': '\u2265',
        'GreaterEqualLess;': '\u22db',
        'GreaterFullEqual;': '\u2267',
        'GreaterGreater;': '\u2aa2',
        'GreaterLess;': '\u2277',
        'GreaterSlantEqual;': '\u2a7e',
        'GreaterTilde;': '\u2273',
        'Gscr;': '\U0001d4a2',
        'gscr;': '\u210a',
        'gsim;': '\u2273',
        'gsime;': '\u2a8e',
        'gsiml;': '\u2a90',
        'GT': '>',
        'gt': '>',
        'GT;': '>',
        'Gt;': '\u226b',
        'gt;': '>',
        'gtcc;': '\u2aa7',
        'gtcir;': '\u2a7a',
        'gtdot;': '\u22d7',
        'gtlPar;': '\u2995',
        'gtquest;': '\u2a7c',
        'gtrapprox;': '\u2a86',
        'gtrarr;': '\u2978',
        'gtrdot;': '\u22d7',
        'gtreqless;': '\u22db',
        'gtreqqless;': '\u2a8c',
        'gtrless;': '\u2277',
        'gtrsim;': '\u2273',
        'gvertneqq;': '\u2269\ufe00',
        'gvnE;': '\u2269\ufe00',
        'Hacek;': '\u02c7',
        'hairsp;': '\u200a',
        'half;': '\xbd',
        'hamilt;': '\u210b',
        'HARDcy;': '\u042a',
        'hardcy;': '\u044a',
        'hArr;': '\u21d4',
        'harr;': '\u2194',
        'harrcir;': '\u2948',
        'harrw;': '\u21ad',
        'Hat;': '^',
        'hbar;': '\u210f',
        'Hcirc;': '\u0124',
        'hcirc;': '\u0125',
        'hearts;': '\u2665',
        'heartsuit;': '\u2665',
        'hellip;': '\u2026',
        'hercon;': '\u22b9',
        'Hfr;': '\u210c',
        'hfr;': '\U0001d525',
        'HilbertSpace;': '\u210b',
        'hksearow;': '\u2925',
        'hkswarow;': '\u2926',
        'hoarr;': '\u21ff',
        'homtht;': '\u223b',
        'hookleftarrow;': '\u21a9',
        'hookrightarrow;': '\u21aa',
        'Hopf;': '\u210d',
        'hopf;': '\U0001d559',
        'horbar;': '\u2015',
        'HorizontalLine;': '\u2500',
        'Hscr;': '\u210b',
        'hscr;': '\U0001d4bd',
        'hslash;': '\u210f',
        'Hstrok;': '\u0126',
        'hstrok;': '\u0127',
        'HumpDownHump;': '\u224e',
        'HumpEqual;': '\u224f',
        'hybull;': '\u2043',
        'hyphen;': '\u2010',
        'Iacute': '\xcd',
        'iacute': '\xed',
        'Iacute;': '\xcd',
        'iacute;': '\xed',
        'ic;': '\u2063',
        'Icirc': '\xce',
        'icirc': '\xee',
        'Icirc;': '\xce',
        'icirc;': '\xee',
        'Icy;': '\u0418',
        'icy;': '\u0438',
        'Idot;': '\u0130',
        'IEcy;': '\u0415',
        'iecy;': '\u0435',
        'iexcl': '\xa1',
        'iexcl;': '\xa1',
        'iff;': '\u21d4',
        'Ifr;': '\u2111',
        'ifr;': '\U0001d526',
        'Igrave': '\xcc',
        'igrave': '\xec',
        'Igrave;': '\xcc',
        'igrave;': '\xec',
        'ii;': '\u2148',
        'iiiint;': '\u2a0c',
        'iiint;': '\u222d',
        'iinfin;': '\u29dc',
        'iiota;': '\u2129',
        'IJlig;': '\u0132',
        'ijlig;': '\u0133',
        'Im;': '\u2111',
        'Imacr;': '\u012a',
        'imacr;': '\u012b',
        'image;': '\u2111',
        'ImaginaryI;': '\u2148',
        'imagline;': '\u2110',
        'imagpart;': '\u2111',
        'imath;': '\u0131',
        'imof;': '\u22b7',
        'imped;': '\u01b5',
        'Implies;': '\u21d2',
        'in;': '\u2208',
        'incare;': '\u2105',
        'infin;': '\u221e',
        'infintie;': '\u29dd',
        'inodot;': '\u0131',
        'Int;': '\u222c',
        'int;': '\u222b',
        'intcal;': '\u22ba',
        'integers;': '\u2124',
        'Integral;': '\u222b',
        'intercal;': '\u22ba',
        'Intersection;': '\u22c2',
        'intlarhk;': '\u2a17',
        'intprod;': '\u2a3c',
        'InvisibleComma;': '\u2063',
        'InvisibleTimes;': '\u2062',
        'IOcy;': '\u0401',
        'iocy;': '\u0451',
        'Iogon;': '\u012e',
        'iogon;': '\u012f',
        'Iopf;': '\U0001d540',
        'iopf;': '\U0001d55a',
        'Iota;': '\u0399',
        'iota;': '\u03b9',
        'iprod;': '\u2a3c',
        'iquest': '\xbf',
        'iquest;': '\xbf',
        'Iscr;': '\u2110',
        'iscr;': '\U0001d4be',
        'isin;': '\u2208',
        'isindot;': '\u22f5',
        'isinE;': '\u22f9',
        'isins;': '\u22f4',
        'isinsv;': '\u22f3',
        'isinv;': '\u2208',
        'it;': '\u2062',
        'Itilde;': '\u0128',
        'itilde;': '\u0129',
        'Iukcy;': '\u0406',
        'iukcy;': '\u0456',
        'Iuml': '\xcf',
        'iuml': '\xef',
        'Iuml;': '\xcf',
        'iuml;': '\xef',
        'Jcirc;': '\u0134',
        'jcirc;': '\u0135',
        'Jcy;': '\u0419',
        'jcy;': '\u0439',
        'Jfr;': '\U0001d50d',
        'jfr;': '\U0001d527',
        'jmath;': '\u0237',
        'Jopf;': '\U0001d541',
        'jopf;': '\U0001d55b',
        'Jscr;': '\U0001d4a5',
        'jscr;': '\U0001d4bf',
        'Jsercy;': '\u0408',
        'jsercy;': '\u0458',
        'Jukcy;': '\u0404',
        'jukcy;': '\u0454',
        'Kappa;': '\u039a',
        'kappa;': '\u03ba',
        'kappav;': '\u03f0',
        'Kcedil;': '\u0136',
        'kcedil;': '\u0137',
        'Kcy;': '\u041a',
        'kcy;': '\u043a',
        'Kfr;': '\U0001d50e',
        'kfr;': '\U0001d528',
        'kgreen;': '\u0138',
        'KHcy;': '\u0425',
        'khcy;': '\u0445',
        'KJcy;': '\u040c',
        'kjcy;': '\u045c',
        'Kopf;': '\U0001d542',
        'kopf;': '\U0001d55c',
        'Kscr;': '\U0001d4a6',
        'kscr;': '\U0001d4c0',
        'lAarr;': '\u21da',
        'Lacute;': '\u0139',
        'lacute;': '\u013a',
        'laemptyv;': '\u29b4',
        'lagran;': '\u2112',
        'Lambda;': '\u039b',
        'lambda;': '\u03bb',
        'Lang;': '\u27ea',
        'lang;': '\u27e8',
        'langd;': '\u2991',
        'langle;': '\u27e8',
        'lap;': '\u2a85',
        'Laplacetrf;': '\u2112',
        'laquo': '\xab',
        'laquo;': '\xab',
        'Larr;': '\u219e',
        'lArr;': '\u21d0',
        'larr;': '\u2190',
        'larrb;': '\u21e4',
        'larrbfs;': '\u291f',
        'larrfs;': '\u291d',
        'larrhk;': '\u21a9',
        'larrlp;': '\u21ab',
        'larrpl;': '\u2939',
        'larrsim;': '\u2973',
        'larrtl;': '\u21a2',
        'lat;': '\u2aab',
        'lAtail;': '\u291b',
        'latail;': '\u2919',
        'late;': '\u2aad',
        'lates;': '\u2aad\ufe00',
        'lBarr;': '\u290e',
        'lbarr;': '\u290c',
        'lbbrk;': '\u2772',
        'lbrace;': '{',
        'lbrack;': '[',
        'lbrke;': '\u298b',
        'lbrksld;': '\u298f',
        'lbrkslu;': '\u298d',
        'Lcaron;': '\u013d',
        'lcaron;': '\u013e',
        'Lcedil;': '\u013b',
        'lcedil;': '\u013c',
        'lceil;': '\u2308',
        'lcub;': '{',
        'Lcy;': '\u041b',
        'lcy;': '\u043b',
        'ldca;': '\u2936',
        'ldquo;': '\u201c',
        'ldquor;': '\u201e',
        'ldrdhar;': '\u2967',
        'ldrushar;': '\u294b',
        'ldsh;': '\u21b2',
        'lE;': '\u2266',
        'le;': '\u2264',
        'LeftAngleBracket;': '\u27e8',
        'LeftArrow;': '\u2190',
        'Leftarrow;': '\u21d0',
        'leftarrow;': '\u2190',
        'LeftArrowBar;': '\u21e4',
        'LeftArrowRightArrow;': '\u21c6',
        'leftarrowtail;': '\u21a2',
        'LeftCeiling;': '\u2308',
        'LeftDoubleBracket;': '\u27e6',
        'LeftDownTeeVector;': '\u2961',
        'LeftDownVector;': '\u21c3',
        'LeftDownVectorBar;': '\u2959',
        'LeftFloor;': '\u230a',
        'leftharpoondown;': '\u21bd',
        'leftharpoonup;': '\u21bc',
        'leftleftarrows;': '\u21c7',
        'LeftRightArrow;': '\u2194',
        'Leftrightarrow;': '\u21d4',
        'leftrightarrow;': '\u2194',
        'leftrightarrows;': '\u21c6',
        'leftrightharpoons;': '\u21cb',
        'leftrightsquigarrow;': '\u21ad',
        'LeftRightVector;': '\u294e',
        'LeftTee;': '\u22a3',
        'LeftTeeArrow;': '\u21a4',
        'LeftTeeVector;': '\u295a',
        'leftthreetimes;': '\u22cb',
        'LeftTriangle;': '\u22b2',
        'LeftTriangleBar;': '\u29cf',
        'LeftTriangleEqual;': '\u22b4',
        'LeftUpDownVector;': '\u2951',
        'LeftUpTeeVector;': '\u2960',
        'LeftUpVector;': '\u21bf',
        'LeftUpVectorBar;': '\u2958',
        'LeftVector;': '\u21bc',
        'LeftVectorBar;': '\u2952',
        'lEg;': '\u2a8b',
        'leg;': '\u22da',
        'leq;': '\u2264',
        'leqq;': '\u2266',
        'leqslant;': '\u2a7d',
        'les;': '\u2a7d',
        'lescc;': '\u2aa8',
        'lesdot;': '\u2a7f',
        'lesdoto;': '\u2a81',
        'lesdotor;': '\u2a83',
        'lesg;': '\u22da\ufe00',
        'lesges;': '\u2a93',
        'lessapprox;': '\u2a85',
        'lessdot;': '\u22d6',
        'lesseqgtr;': '\u22da',
        'lesseqqgtr;': '\u2a8b',
        'LessEqualGreater;': '\u22da',
        'LessFullEqual;': '\u2266',
        'LessGreater;': '\u2276',
        'lessgtr;': '\u2276',
        'LessLess;': '\u2aa1',
        'lesssim;': '\u2272',
        'LessSlantEqual;': '\u2a7d',
        'LessTilde;': '\u2272',
        'lfisht;': '\u297c',
        'lfloor;': '\u230a',
        'Lfr;': '\U0001d50f',
        'lfr;': '\U0001d529',
        'lg;': '\u2276',
        'lgE;': '\u2a91',
        'lHar;': '\u2962',
        'lhard;': '\u21bd',
        'lharu;': '\u21bc',
        'lharul;': '\u296a',
        'lhblk;': '\u2584',
        'LJcy;': '\u0409',
        'ljcy;': '\u0459',
        'Ll;': '\u22d8',
        'll;': '\u226a',
        'llarr;': '\u21c7',
        'llcorner;': '\u231e',
        'Lleftarrow;': '\u21da',
        'llhard;': '\u296b',
        'lltri;': '\u25fa',
        'Lmidot;': '\u013f',
        'lmidot;': '\u0140',
        'lmoust;': '\u23b0',
        'lmoustache;': '\u23b0',
        'lnap;': '\u2a89',
        'lnapprox;': '\u2a89',
        'lnE;': '\u2268',
        'lne;': '\u2a87',
        'lneq;': '\u2a87',
        'lneqq;': '\u2268',
        'lnsim;': '\u22e6',
        'loang;': '\u27ec',
        'loarr;': '\u21fd',
        'lobrk;': '\u27e6',
        'LongLeftArrow;': '\u27f5',
        'Longleftarrow;': '\u27f8',
        'longleftarrow;': '\u27f5',
        'LongLeftRightArrow;': '\u27f7',
        'Longleftrightarrow;': '\u27fa',
        'longleftrightarrow;': '\u27f7',
        'longmapsto;': '\u27fc',
        'LongRightArrow;': '\u27f6',
        'Longrightarrow;': '\u27f9',
        'longrightarrow;': '\u27f6',
        'looparrowleft;': '\u21ab',
        'looparrowright;': '\u21ac',
        'lopar;': '\u2985',
        'Lopf;': '\U0001d543',
        'lopf;': '\U0001d55d',
        'loplus;': '\u2a2d',
        'lotimes;': '\u2a34',
        'lowast;': '\u2217',
        'lowbar;': '_',
        'LowerLeftArrow;': '\u2199',
        'LowerRightArrow;': '\u2198',
        'loz;': '\u25ca',
        'lozenge;': '\u25ca',
        'lozf;': '\u29eb',
        'lpar;': '(',
        'lparlt;': '\u2993',
        'lrarr;': '\u21c6',
        'lrcorner;': '\u231f',
        'lrhar;': '\u21cb',
        'lrhard;': '\u296d',
        'lrm;': '\u200e',
        'lrtri;': '\u22bf',
        'lsaquo;': '\u2039',
        'Lscr;': '\u2112',
        'lscr;': '\U0001d4c1',
        'Lsh;': '\u21b0',
        'lsh;': '\u21b0',
        'lsim;': '\u2272',
        'lsime;': '\u2a8d',
        'lsimg;': '\u2a8f',
        'lsqb;': '[',
        'lsquo;': '\u2018',
        'lsquor;': '\u201a',
        'Lstrok;': '\u0141',
        'lstrok;': '\u0142',
        'LT': '<',
        'lt': '<',
        'LT;': '<',
        'Lt;': '\u226a',
        'lt;': '<',
        'ltcc;': '\u2aa6',
        'ltcir;': '\u2a79',
        'ltdot;': '\u22d6',
        'lthree;': '\u22cb',
        'ltimes;': '\u22c9',
        'ltlarr;': '\u2976',
        'ltquest;': '\u2a7b',
        'ltri;': '\u25c3',
        'ltrie;': '\u22b4',
        'ltrif;': '\u25c2',
        'ltrPar;': '\u2996',
        'lurdshar;': '\u294a',
        'luruhar;': '\u2966',
        'lvertneqq;': '\u2268\ufe00',
        'lvnE;': '\u2268\ufe00',
        'macr': '\xaf',
        'macr;': '\xaf',
        'male;': '\u2642',
        'malt;': '\u2720',
        'maltese;': '\u2720',
        'Map;': '\u2905',
        'map;': '\u21a6',
        'mapsto;': '\u21a6',
        'mapstodown;': '\u21a7',
        'mapstoleft;': '\u21a4',
        'mapstoup;': '\u21a5',
        'marker;': '\u25ae',
        'mcomma;': '\u2a29',
        'Mcy;': '\u041c',
        'mcy;': '\u043c',
        'mdash;': '\u2014',
        'mDDot;': '\u223a',
        'measuredangle;': '\u2221',
        'MediumSpace;': '\u205f',
        'Mellintrf;': '\u2133',
        'Mfr;': '\U0001d510',
        'mfr;': '\U0001d52a',
        'mho;': '\u2127',
        'micro': '\xb5',
        'micro;': '\xb5',
        'mid;': '\u2223',
        'midast;': '*',
        'midcir;': '\u2af0',
        'middot': '\xb7',
        'middot;': '\xb7',
        'minus;': '\u2212',
        'minusb;': '\u229f',
        'minusd;': '\u2238',
        'minusdu;': '\u2a2a',
        'MinusPlus;': '\u2213',
        'mlcp;': '\u2adb',
        'mldr;': '\u2026',
        'mnplus;': '\u2213',
        'models;': '\u22a7',
        'Mopf;': '\U0001d544',
        'mopf;': '\U0001d55e',
        'mp;': '\u2213',
        'Mscr;': '\u2133',
        'mscr;': '\U0001d4c2',
        'mstpos;': '\u223e',
        'Mu;': '\u039c',
        'mu;': '\u03bc',
        'multimap;': '\u22b8',
        'mumap;': '\u22b8',
        'nabla;': '\u2207',
        'Nacute;': '\u0143',
        'nacute;': '\u0144',
        'nang;': '\u2220\u20d2',
        'nap;': '\u2249',
        'napE;': '\u2a70\u0338',
        'napid;': '\u224b\u0338',
        'napos;': '\u0149',
        'napprox;': '\u2249',
        'natur;': '\u266e',
        'natural;': '\u266e',
        'naturals;': '\u2115',
        'nbsp': '\xa0',
        'nbsp;': '\xa0',
        'nbump;': '\u224e\u0338',
        'nbumpe;': '\u224f\u0338',
        'ncap;': '\u2a43',
        'Ncaron;': '\u0147',
        'ncaron;': '\u0148',
        'Ncedil;': '\u0145',
        'ncedil;': '\u0146',
        'ncong;': '\u2247',
        'ncongdot;': '\u2a6d\u0338',
        'ncup;': '\u2a42',
        'Ncy;': '\u041d',
        'ncy;': '\u043d',
        'ndash;': '\u2013',
        'ne;': '\u2260',
        'nearhk;': '\u2924',
        'neArr;': '\u21d7',
        'nearr;': '\u2197',
        'nearrow;': '\u2197',
        'nedot;': '\u2250\u0338',
        'NegativeMediumSpace;': '\u200b',
        'NegativeThickSpace;': '\u200b',
        'NegativeThinSpace;': '\u200b',
        'NegativeVeryThinSpace;': '\u200b',
        'nequiv;': '\u2262',
        'nesear;': '\u2928',
        'nesim;': '\u2242\u0338',
        'NestedGreaterGreater;': '\u226b',
        'NestedLessLess;': '\u226a',
        'NewLine;': '\n',
        'nexist;': '\u2204',
        'nexists;': '\u2204',
        'Nfr;': '\U0001d511',
        'nfr;': '\U0001d52b',
        'ngE;': '\u2267\u0338',
        'nge;': '\u2271',
        'ngeq;': '\u2271',
        'ngeqq;': '\u2267\u0338',
        'ngeqslant;': '\u2a7e\u0338',
        'nges;': '\u2a7e\u0338',
        'nGg;': '\u22d9\u0338',
        'ngsim;': '\u2275',
        'nGt;': '\u226b\u20d2',
        'ngt;': '\u226f',
        'ngtr;': '\u226f',
        'nGtv;': '\u226b\u0338',
        'nhArr;': '\u21ce',
        'nharr;': '\u21ae',
        'nhpar;': '\u2af2',
        'ni;': '\u220b',
        'nis;': '\u22fc',
        'nisd;': '\u22fa',
        'niv;': '\u220b',
        'NJcy;': '\u040a',
        'njcy;': '\u045a',
        'nlArr;': '\u21cd',
        'nlarr;': '\u219a',
        'nldr;': '\u2025',
        'nlE;': '\u2266\u0338',
        'nle;': '\u2270',
        'nLeftarrow;': '\u21cd',
        'nleftarrow;': '\u219a',
        'nLeftrightarrow;': '\u21ce',
        'nleftrightarrow;': '\u21ae',
        'nleq;': '\u2270',
        'nleqq;': '\u2266\u0338',
        'nleqslant;': '\u2a7d\u0338',
        'nles;': '\u2a7d\u0338',
        'nless;': '\u226e',
        'nLl;': '\u22d8\u0338',
        'nlsim;': '\u2274',
        'nLt;': '\u226a\u20d2',
        'nlt;': '\u226e',
        'nltri;': '\u22ea',
        'nltrie;': '\u22ec',
        'nLtv;': '\u226a\u0338',
        'nmid;': '\u2224',
        'NoBreak;': '\u2060',
        'NonBreakingSpace;': '\xa0',
        'Nopf;': '\u2115',
        'nopf;': '\U0001d55f',
        'not': '\xac',
        'Not;': '\u2aec',
        'not;': '\xac',
        'NotCongruent;': '\u2262',
        'NotCupCap;': '\u226d',
        'NotDoubleVerticalBar;': '\u2226',
        'NotElement;': '\u2209',
        'NotEqual;': '\u2260',
        'NotEqualTilde;': '\u2242\u0338',
        'NotExists;': '\u2204',
        'NotGreater;': '\u226f',
        'NotGreaterEqual;': '\u2271',
        'NotGreaterFullEqual;': '\u2267\u0338',
        'NotGreaterGreater;': '\u226b\u0338',
        'NotGreaterLess;': '\u2279',
        'NotGreaterSlantEqual;': '\u2a7e\u0338',
        'NotGreaterTilde;': '\u2275',
        'NotHumpDownHump;': '\u224e\u0338',
        'NotHumpEqual;': '\u224f\u0338',
        'notin;': '\u2209',
        'notindot;': '\u22f5\u0338',
        'notinE;': '\u22f9\u0338',
        'notinva;': '\u2209',
        'notinvb;': '\u22f7',
        'notinvc;': '\u22f6',
        'NotLeftTriangle;': '\u22ea',
        'NotLeftTriangleBar;': '\u29cf\u0338',
        'NotLeftTriangleEqual;': '\u22ec',
        'NotLess;': '\u226e',
        'NotLessEqual;': '\u2270',
        'NotLessGreater;': '\u2278',
        'NotLessLess;': '\u226a\u0338',
        'NotLessSlantEqual;': '\u2a7d\u0338',
        'NotLessTilde;': '\u2274',
        'NotNestedGreaterGreater;': '\u2aa2\u0338',
        'NotNestedLessLess;': '\u2aa1\u0338',
        'notni;': '\u220c',
        'notniva;': '\u220c',
        'notnivb;': '\u22fe',
        'notnivc;': '\u22fd',
        'NotPrecedes;': '\u2280',
        'NotPrecedesEqual;': '\u2aaf\u0338',
        'NotPrecedesSlantEqual;': '\u22e0',
        'NotReverseElement;': '\u220c',
        'NotRightTriangle;': '\u22eb',
        'NotRightTriangleBar;': '\u29d0\u0338',
        'NotRightTriangleEqual;': '\u22ed',
        'NotSquareSubset;': '\u228f\u0338',
        'NotSquareSubsetEqual;': '\u22e2',
        'NotSquareSuperset;': '\u2290\u0338',
        'NotSquareSupersetEqual;': '\u22e3',
        'NotSubset;': '\u2282\u20d2',
        'NotSubsetEqual;': '\u2288',
        'NotSucceeds;': '\u2281',
        'NotSucceedsEqual;': '\u2ab0\u0338',
        'NotSucceedsSlantEqual;': '\u22e1',
        'NotSucceedsTilde;': '\u227f\u0338',
        'NotSuperset;': '\u2283\u20d2',
        'NotSupersetEqual;': '\u2289',
        'NotTilde;': '\u2241',
        'NotTildeEqual;': '\u2244',
        'NotTildeFullEqual;': '\u2247',
        'NotTildeTilde;': '\u2249',
        'NotVerticalBar;': '\u2224',
        'npar;': '\u2226',
        'nparallel;': '\u2226',
        'nparsl;': '\u2afd\u20e5',
        'npart;': '\u2202\u0338',
        'npolint;': '\u2a14',
        'npr;': '\u2280',
        'nprcue;': '\u22e0',
        'npre;': '\u2aaf\u0338',
        'nprec;': '\u2280',
        'npreceq;': '\u2aaf\u0338',
        'nrArr;': '\u21cf',
        'nrarr;': '\u219b',
        'nrarrc;': '\u2933\u0338',
        'nrarrw;': '\u219d\u0338',
        'nRightarrow;': '\u21cf',
        'nrightarrow;': '\u219b',
        'nrtri;': '\u22eb',
        'nrtrie;': '\u22ed',
        'nsc;': '\u2281',
        'nsccue;': '\u22e1',
        'nsce;': '\u2ab0\u0338',
        'Nscr;': '\U0001d4a9',
        'nscr;': '\U0001d4c3',
        'nshortmid;': '\u2224',
        'nshortparallel;': '\u2226',
        'nsim;': '\u2241',
        'nsime;': '\u2244',
        'nsimeq;': '\u2244',
        'nsmid;': '\u2224',
        'nspar;': '\u2226',
        'nsqsube;': '\u22e2',
        'nsqsupe;': '\u22e3',
        'nsub;': '\u2284',
        'nsubE;': '\u2ac5\u0338',
        'nsube;': '\u2288',
        'nsubset;': '\u2282\u20d2',
        'nsubseteq;': '\u2288',
        'nsubseteqq;': '\u2ac5\u0338',
        'nsucc;': '\u2281',
        'nsucceq;': '\u2ab0\u0338',
        'nsup;': '\u2285',
        'nsupE;': '\u2ac6\u0338',
        'nsupe;': '\u2289',
        'nsupset;': '\u2283\u20d2',
        'nsupseteq;': '\u2289',
        'nsupseteqq;': '\u2ac6\u0338',
        'ntgl;': '\u2279',
        'Ntilde': '\xd1',
        'ntilde': '\xf1',
        'Ntilde;': '\xd1',
        'ntilde;': '\xf1',
        'ntlg;': '\u2278',
        'ntriangleleft;': '\u22ea',
        'ntrianglelefteq;': '\u22ec',
        'ntriangleright;': '\u22eb',
        'ntrianglerighteq;': '\u22ed',
        'Nu;': '\u039d',
        'nu;': '\u03bd',
        'num;': '#',
        'numero;': '\u2116',
        'numsp;': '\u2007',
        'nvap;': '\u224d\u20d2',
        'nVDash;': '\u22af',
        'nVdash;': '\u22ae',
        'nvDash;': '\u22ad',
        'nvdash;': '\u22ac',
        'nvge;': '\u2265\u20d2',
        'nvgt;': '>\u20d2',
        'nvHarr;': '\u2904',
        'nvinfin;': '\u29de',
        'nvlArr;': '\u2902',
        'nvle;': '\u2264\u20d2',
        'nvlt;': '<\u20d2',
        'nvltrie;': '\u22b4\u20d2',
        'nvrArr;': '\u2903',
        'nvrtrie;': '\u22b5\u20d2',
        'nvsim;': '\u223c\u20d2',
        'nwarhk;': '\u2923',
        'nwArr;': '\u21d6',
        'nwarr;': '\u2196',
        'nwarrow;': '\u2196',
        'nwnear;': '\u2927',
        'Oacute': '\xd3',
        'oacute': '\xf3',
        'Oacute;': '\xd3',
        'oacute;': '\xf3',
        'oast;': '\u229b',
        'ocir;': '\u229a',
        'Ocirc': '\xd4',
        'ocirc': '\xf4',
        'Ocirc;': '\xd4',
        'ocirc;': '\xf4',
        'Ocy;': '\u041e',
        'ocy;': '\u043e',
        'odash;': '\u229d',
        'Odblac;': '\u0150',
        'odblac;': '\u0151',
        'odiv;': '\u2a38',
        'odot;': '\u2299',
        'odsold;': '\u29bc',
        'OElig;': '\u0152',
        'oelig;': '\u0153',
        'ofcir;': '\u29bf',
        'Ofr;': '\U0001d512',
        'ofr;': '\U0001d52c',
        'ogon;': '\u02db',
        'Ograve': '\xd2',
        'ograve': '\xf2',
        'Ograve;': '\xd2',
        'ograve;': '\xf2',
        'ogt;': '\u29c1',
        'ohbar;': '\u29b5',
        'ohm;': '\u03a9',
        'oint;': '\u222e',
        'olarr;': '\u21ba',
        'olcir;': '\u29be',
        'olcross;': '\u29bb',
        'oline;': '\u203e',
        'olt;': '\u29c0',
        'Omacr;': '\u014c',
        'omacr;': '\u014d',
        'Omega;': '\u03a9',
        'omega;': '\u03c9',
        'Omicron;': '\u039f',
        'omicron;': '\u03bf',
        'omid;': '\u29b6',
        'ominus;': '\u2296',
        'Oopf;': '\U0001d546',
        'oopf;': '\U0001d560',
        'opar;': '\u29b7',
        'OpenCurlyDoubleQuote;': '\u201c',
        'OpenCurlyQuote;': '\u2018',
        'operp;': '\u29b9',
        'oplus;': '\u2295',
        'Or;': '\u2a54',
        'or;': '\u2228',
        'orarr;': '\u21bb',
        'ord;': '\u2a5d',
        'order;': '\u2134',
        'orderof;': '\u2134',
        'ordf': '\xaa',
        'ordf;': '\xaa',
        'ordm': '\xba',
        'ordm;': '\xba',
        'origof;': '\u22b6',
        'oror;': '\u2a56',
        'orslope;': '\u2a57',
        'orv;': '\u2a5b',
        'oS;': '\u24c8',
        'Oscr;': '\U0001d4aa',
        'oscr;': '\u2134',
        'Oslash': '\xd8',
        'oslash': '\xf8',
        'Oslash;': '\xd8',
        'oslash;': '\xf8',
        'osol;': '\u2298',
        'Otilde': '\xd5',
        'otilde': '\xf5',
        'Otilde;': '\xd5',
        'otilde;': '\xf5',
        'Otimes;': '\u2a37',
        'otimes;': '\u2297',
        'otimesas;': '\u2a36',
        'Ouml': '\xd6',
        'ouml': '\xf6',
        'Ouml;': '\xd6',
        'ouml;': '\xf6',
        'ovbar;': '\u233d',
        'OverBar;': '\u203e',
        'OverBrace;': '\u23de',
        'OverBracket;': '\u23b4',
        'OverParenthesis;': '\u23dc',
        'par;': '\u2225',
        'para': '\xb6',
        'para;': '\xb6',
        'parallel;': '\u2225',
        'parsim;': '\u2af3',
        'parsl;': '\u2afd',
        'part;': '\u2202',
        'PartialD;': '\u2202',
        'Pcy;': '\u041f',
        'pcy;': '\u043f',
        'percnt;': '%',
        'period;': '.',
        'permil;': '\u2030',
        'perp;': '\u22a5',
        'pertenk;': '\u2031',
        'Pfr;': '\U0001d513',
        'pfr;': '\U0001d52d',
        'Phi;': '\u03a6',
        'phi;': '\u03c6',
        'phiv;': '\u03d5',
        'phmmat;': '\u2133',
        'phone;': '\u260e',
        'Pi;': '\u03a0',
        'pi;': '\u03c0',
        'pitchfork;': '\u22d4',
        'piv;': '\u03d6',
        'planck;': '\u210f',
        'planckh;': '\u210e',
        'plankv;': '\u210f',
        'plus;': '+',
        'plusacir;': '\u2a23',
        'plusb;': '\u229e',
        'pluscir;': '\u2a22',
        'plusdo;': '\u2214',
        'plusdu;': '\u2a25',
        'pluse;': '\u2a72',
        'PlusMinus;': '\xb1',
        'plusmn': '\xb1',
        'plusmn;': '\xb1',
        'plussim;': '\u2a26',
        'plustwo;': '\u2a27',
        'pm;': '\xb1',
        'Poincareplane;': '\u210c',
        'pointint;': '\u2a15',
        'Popf;': '\u2119',
        'popf;': '\U0001d561',
        'pound': '\xa3',
        'pound;': '\xa3',
        'Pr;': '\u2abb',
        'pr;': '\u227a',
        'prap;': '\u2ab7',
        'prcue;': '\u227c',
        'prE;': '\u2ab3',
        'pre;': '\u2aaf',
        'prec;': '\u227a',
        'precapprox;': '\u2ab7',
        'preccurlyeq;': '\u227c',
        'Precedes;': '\u227a',
        'PrecedesEqual;': '\u2aaf',
        'PrecedesSlantEqual;': '\u227c',
        'PrecedesTilde;': '\u227e',
        'preceq;': '\u2aaf',
        'precnapprox;': '\u2ab9',
        'precneqq;': '\u2ab5',
        'precnsim;': '\u22e8',
        'precsim;': '\u227e',
        'Prime;': '\u2033',
        'prime;': '\u2032',
        'primes;': '\u2119',
        'prnap;': '\u2ab9',
        'prnE;': '\u2ab5',
        'prnsim;': '\u22e8',
        'prod;': '\u220f',
        'Product;': '\u220f',
        'profalar;': '\u232e',
        'profline;': '\u2312',
        'profsurf;': '\u2313',
        'prop;': '\u221d',
        'Proportion;': '\u2237',
        'Proportional;': '\u221d',
        'propto;': '\u221d',
        'prsim;': '\u227e',
        'prurel;': '\u22b0',
        'Pscr;': '\U0001d4ab',
        'pscr;': '\U0001d4c5',
        'Psi;': '\u03a8',
        'psi;': '\u03c8',
        'puncsp;': '\u2008',
        'Qfr;': '\U0001d514',
        'qfr;': '\U0001d52e',
        'qint;': '\u2a0c',
        'Qopf;': '\u211a',
        'qopf;': '\U0001d562',
        'qprime;': '\u2057',
        'Qscr;': '\U0001d4ac',
        'qscr;': '\U0001d4c6',
        'quaternions;': '\u210d',
        'quatint;': '\u2a16',
        'quest;': '?',
        'questeq;': '\u225f',
        'QUOT': '"',
        'quot': '"',
        'QUOT;': '"',
        'quot;': '"',
        'rAarr;': '\u21db',
        'race;': '\u223d\u0331',
        'Racute;': '\u0154',
        'racute;': '\u0155',
        'radic;': '\u221a',
        'raemptyv;': '\u29b3',
        'Rang;': '\u27eb',
        'rang;': '\u27e9',
        'rangd;': '\u2992',
        'range;': '\u29a5',
        'rangle;': '\u27e9',
        'raquo': '\xbb',
        'raquo;': '\xbb',
        'Rarr;': '\u21a0',
        'rArr;': '\u21d2',
        'rarr;': '\u2192',
        'rarrap;': '\u2975',
        'rarrb;': '\u21e5',
        'rarrbfs;': '\u2920',
        'rarrc;': '\u2933',
        'rarrfs;': '\u291e',
        'rarrhk;': '\u21aa',
        'rarrlp;': '\u21ac',
        'rarrpl;': '\u2945',
        'rarrsim;': '\u2974',
        'Rarrtl;': '\u2916',
        'rarrtl;': '\u21a3',
        'rarrw;': '\u219d',
        'rAtail;': '\u291c',
        'ratail;': '\u291a',
        'ratio;': '\u2236',
        'rationals;': '\u211a',
        'RBarr;': '\u2910',
        'rBarr;': '\u290f',
        'rbarr;': '\u290d',
        'rbbrk;': '\u2773',
        'rbrace;': '}',
        'rbrack;': ']',
        'rbrke;': '\u298c',
        'rbrksld;': '\u298e',
        'rbrkslu;': '\u2990',
        'Rcaron;': '\u0158',
        'rcaron;': '\u0159',
        'Rcedil;': '\u0156',
        'rcedil;': '\u0157',
        'rceil;': '\u2309',
        'rcub;': '}',
        'Rcy;': '\u0420',
        'rcy;': '\u0440',
        'rdca;': '\u2937',
        'rdldhar;': '\u2969',
        'rdquo;': '\u201d',
        'rdquor;': '\u201d',
        'rdsh;': '\u21b3',
        'Re;': '\u211c',
        'real;': '\u211c',
        'realine;': '\u211b',
        'realpart;': '\u211c',
        'reals;': '\u211d',
        'rect;': '\u25ad',
        'REG': '\xae',
        'reg': '\xae',
        'REG;': '\xae',
        'reg;': '\xae',
        'ReverseElement;': '\u220b',
        'ReverseEquilibrium;': '\u21cb',
        'ReverseUpEquilibrium;': '\u296f',
        'rfisht;': '\u297d',
        'rfloor;': '\u230b',
        'Rfr;': '\u211c',
        'rfr;': '\U0001d52f',
        'rHar;': '\u2964',
        'rhard;': '\u21c1',
        'rharu;': '\u21c0',
        'rharul;': '\u296c',
        'Rho;': '\u03a1',
        'rho;': '\u03c1',
        'rhov;': '\u03f1',
        'RightAngleBracket;': '\u27e9',
        'RightArrow;': '\u2192',
        'Rightarrow;': '\u21d2',
        'rightarrow;': '\u2192',
        'RightArrowBar;': '\u21e5',
        'RightArrowLeftArrow;': '\u21c4',
        'rightarrowtail;': '\u21a3',
        'RightCeiling;': '\u2309',
        'RightDoubleBracket;': '\u27e7',
        'RightDownTeeVector;': '\u295d',
        'RightDownVector;': '\u21c2',
        'RightDownVectorBar;': '\u2955',
        'RightFloor;': '\u230b',
        'rightharpoondown;': '\u21c1',
        'rightharpoonup;': '\u21c0',
        'rightleftarrows;': '\u21c4',
        'rightleftharpoons;': '\u21cc',
        'rightrightarrows;': '\u21c9',
        'rightsquigarrow;': '\u219d',
        'RightTee;': '\u22a2',
        'RightTeeArrow;': '\u21a6',
        'RightTeeVector;': '\u295b',
        'rightthreetimes;': '\u22cc',
        'RightTriangle;': '\u22b3',
        'RightTriangleBar;': '\u29d0',
        'RightTriangleEqual;': '\u22b5',
        'RightUpDownVector;': '\u294f',
        'RightUpTeeVector;': '\u295c',
        'RightUpVector;': '\u21be',
        'RightUpVectorBar;': '\u2954',
        'RightVector;': '\u21c0',
        'RightVectorBar;': '\u2953',
        'ring;': '\u02da',
        'risingdotseq;': '\u2253',
        'rlarr;': '\u21c4',
        'rlhar;': '\u21cc',
        'rlm;': '\u200f',
        'rmoust;': '\u23b1',
        'rmoustache;': '\u23b1',
        'rnmid;': '\u2aee',
        'roang;': '\u27ed',
        'roarr;': '\u21fe',
        'robrk;': '\u27e7',
        'ropar;': '\u2986',
        'Ropf;': '\u211d',
        'ropf;': '\U0001d563',
        'roplus;': '\u2a2e',
        'rotimes;': '\u2a35',
        'RoundImplies;': '\u2970',
        'rpar;': ')',
        'rpargt;': '\u2994',
        'rppolint;': '\u2a12',
        'rrarr;': '\u21c9',
        'Rrightarrow;': '\u21db',
        'rsaquo;': '\u203a',
        'Rscr;': '\u211b',
        'rscr;': '\U0001d4c7',
        'Rsh;': '\u21b1',
        'rsh;': '\u21b1',
        'rsqb;': ']',
        'rsquo;': '\u2019',
        'rsquor;': '\u2019',
        'rthree;': '\u22cc',
        'rtimes;': '\u22ca',
        'rtri;': '\u25b9',
        'rtrie;': '\u22b5',
        'rtrif;': '\u25b8',
        'rtriltri;': '\u29ce',
        'RuleDelayed;': '\u29f4',
        'ruluhar;': '\u2968',
        'rx;': '\u211e',
        'Sacute;': '\u015a',
        'sacute;': '\u015b',
        'sbquo;': '\u201a',
        'Sc;': '\u2abc',
        'sc;': '\u227b',
        'scap;': '\u2ab8',
        'Scaron;': '\u0160',
        'scaron;': '\u0161',
        'sccue;': '\u227d',
        'scE;': '\u2ab4',
        'sce;': '\u2ab0',
        'Scedil;': '\u015e',
        'scedil;': '\u015f',
        'Scirc;': '\u015c',
        'scirc;': '\u015d',
        'scnap;': '\u2aba',
        'scnE;': '\u2ab6',
        'scnsim;': '\u22e9',
        'scpolint;': '\u2a13',
        'scsim;': '\u227f',
        'Scy;': '\u0421',
        'scy;': '\u0441',
        'sdot;': '\u22c5',
        'sdotb;': '\u22a1',
        'sdote;': '\u2a66',
        'searhk;': '\u2925',
        'seArr;': '\u21d8',
        'searr;': '\u2198',
        'searrow;': '\u2198',
        'sect': '\xa7',
        'sect;': '\xa7',
        'semi;': ';',
        'seswar;': '\u2929',
        'setminus;': '\u2216',
        'setmn;': '\u2216',
        'sext;': '\u2736',
        'Sfr;': '\U0001d516',
        'sfr;': '\U0001d530',
        'sfrown;': '\u2322',
        'sharp;': '\u266f',
        'SHCHcy;': '\u0429',
        'shchcy;': '\u0449',
        'SHcy;': '\u0428',
        'shcy;': '\u0448',
        'ShortDownArrow;': '\u2193',
        'ShortLeftArrow;': '\u2190',
        'shortmid;': '\u2223',
        'shortparallel;': '\u2225',
        'ShortRightArrow;': '\u2192',
        'ShortUpArrow;': '\u2191',
        'shy': '\xad',
        'shy;': '\xad',
        'Sigma;': '\u03a3',
        'sigma;': '\u03c3',
        'sigmaf;': '\u03c2',
        'sigmav;': '\u03c2',
        'sim;': '\u223c',
        'simdot;': '\u2a6a',
        'sime;': '\u2243',
        'simeq;': '\u2243',
        'simg;': '\u2a9e',
        'simgE;': '\u2aa0',
        'siml;': '\u2a9d',
        'simlE;': '\u2a9f',
        'simne;': '\u2246',
        'simplus;': '\u2a24',
        'simrarr;': '\u2972',
        'slarr;': '\u2190',
        'SmallCircle;': '\u2218',
        'smallsetminus;': '\u2216',
        'smashp;': '\u2a33',
        'smeparsl;': '\u29e4',
        'smid;': '\u2223',
        'smile;': '\u2323',
        'smt;': '\u2aaa',
        'smte;': '\u2aac',
        'smtes;': '\u2aac\ufe00',
        'SOFTcy;': '\u042c',
        'softcy;': '\u044c',
        'sol;': '/',
        'solb;': '\u29c4',
        'solbar;': '\u233f',
        'Sopf;': '\U0001d54a',
        'sopf;': '\U0001d564',
        'spades;': '\u2660',
        'spadesuit;': '\u2660',
        'spar;': '\u2225',
        'sqcap;': '\u2293',
        'sqcaps;': '\u2293\ufe00',
        'sqcup;': '\u2294',
        'sqcups;': '\u2294\ufe00',
        'Sqrt;': '\u221a',
        'sqsub;': '\u228f',
        'sqsube;': '\u2291',
        'sqsubset;': '\u228f',
        'sqsubseteq;': '\u2291',
        'sqsup;': '\u2290',
        'sqsupe;': '\u2292',
        'sqsupset;': '\u2290',
        'sqsupseteq;': '\u2292',
        'squ;': '\u25a1',
        'Square;': '\u25a1',
        'square;': '\u25a1',
        'SquareIntersection;': '\u2293',
        'SquareSubset;': '\u228f',
        'SquareSubsetEqual;': '\u2291',
        'SquareSuperset;': '\u2290',
        'SquareSupersetEqual;': '\u2292',
        'SquareUnion;': '\u2294',
        'squarf;': '\u25aa',
        'squf;': '\u25aa',
        'srarr;': '\u2192',
        'Sscr;': '\U0001d4ae',
        'sscr;': '\U0001d4c8',
        'ssetmn;': '\u2216',
        'ssmile;': '\u2323',
        'sstarf;': '\u22c6',
        'Star;': '\u22c6',
        'star;': '\u2606',
        'starf;': '\u2605',
        'straightepsilon;': '\u03f5',
        'straightphi;': '\u03d5',
        'strns;': '\xaf',
        'Sub;': '\u22d0',
        'sub;': '\u2282',
        'subdot;': '\u2abd',
        'subE;': '\u2ac5',
        'sube;': '\u2286',
        'subedot;': '\u2ac3',
        'submult;': '\u2ac1',
        'subnE;': '\u2acb',
        'subne;': '\u228a',
        'subplus;': '\u2abf',
        'subrarr;': '\u2979',
        'Subset;': '\u22d0',
        'subset;': '\u2282',
        'subseteq;': '\u2286',
        'subseteqq;': '\u2ac5',
        'SubsetEqual;': '\u2286',
        'subsetneq;': '\u228a',
        'subsetneqq;': '\u2acb',
        'subsim;': '\u2ac7',
        'subsub;': '\u2ad5',
        'subsup;': '\u2ad3',
        'succ;': '\u227b',
        'succapprox;': '\u2ab8',
        'succcurlyeq;': '\u227d',
        'Succeeds;': '\u227b',
        'SucceedsEqual;': '\u2ab0',
        'SucceedsSlantEqual;': '\u227d',
        'SucceedsTilde;': '\u227f',
        'succeq;': '\u2ab0',
        'succnapprox;': '\u2aba',
        'succneqq;': '\u2ab6',
        'succnsim;': '\u22e9',
        'succsim;': '\u227f',
        'SuchThat;': '\u220b',
        'Sum;': '\u2211',
        'sum;': '\u2211',
        'sung;': '\u266a',
        'sup1': '\xb9',
        'sup1;': '\xb9',
        'sup2': '\xb2',
        'sup2;': '\xb2',
        'sup3': '\xb3',
        'sup3;': '\xb3',
        'Sup;': '\u22d1',
        'sup;': '\u2283',
        'supdot;': '\u2abe',
        'supdsub;': '\u2ad8',
        'supE;': '\u2ac6',
        'supe;': '\u2287',
        'supedot;': '\u2ac4',
        'Superset;': '\u2283',
        'SupersetEqual;': '\u2287',
        'suphsol;': '\u27c9',
        'suphsub;': '\u2ad7',
        'suplarr;': '\u297b',
        'supmult;': '\u2ac2',
        'supnE;': '\u2acc',
        'supne;': '\u228b',
        'supplus;': '\u2ac0',
        'Supset;': '\u22d1',
        'supset;': '\u2283',
        'supseteq;': '\u2287',
        'supseteqq;': '\u2ac6',
        'supsetneq;': '\u228b',
        'supsetneqq;': '\u2acc',
        'supsim;': '\u2ac8',
        'supsub;': '\u2ad4',
        'supsup;': '\u2ad6',
        'swarhk;': '\u2926',
        'swArr;': '\u21d9',
        'swarr;': '\u2199',
        'swarrow;': '\u2199',
        'swnwar;': '\u292a',
        'szlig': '\xdf',
        'szlig;': '\xdf',
        'Tab;': '\t',
        'target;': '\u2316',
        'Tau;': '\u03a4',
        'tau;': '\u03c4',
        'tbrk;': '\u23b4',
        'Tcaron;': '\u0164',
        'tcaron;': '\u0165',
        'Tcedil;': '\u0162',
        'tcedil;': '\u0163',
        'Tcy;': '\u0422',
        'tcy;': '\u0442',
        'tdot;': '\u20db',
        'telrec;': '\u2315',
        'Tfr;': '\U0001d517',
        'tfr;': '\U0001d531',
        'there4;': '\u2234',
        'Therefore;': '\u2234',
        'therefore;': '\u2234',
        'Theta;': '\u0398',
        'theta;': '\u03b8',
        'thetasym;': '\u03d1',
        'thetav;': '\u03d1',
        'thickapprox;': '\u2248',
        'thicksim;': '\u223c',
        'ThickSpace;': '\u205f\u200a',
        'thinsp;': '\u2009',
        'ThinSpace;': '\u2009',
        'thkap;': '\u2248',
        'thksim;': '\u223c',
        'THORN': '\xde',
        'thorn': '\xfe',
        'THORN;': '\xde',
        'thorn;': '\xfe',
        'Tilde;': '\u223c',
        'tilde;': '\u02dc',
        'TildeEqual;': '\u2243',
        'TildeFullEqual;': '\u2245',
        'TildeTilde;': '\u2248',
        'times': '\xd7',
        'times;': '\xd7',
        'timesb;': '\u22a0',
        'timesbar;': '\u2a31',
        'timesd;': '\u2a30',
        'tint;': '\u222d',
        'toea;': '\u2928',
        'top;': '\u22a4',
        'topbot;': '\u2336',
        'topcir;': '\u2af1',
        'Topf;': '\U0001d54b',
        'topf;': '\U0001d565',
        'topfork;': '\u2ada',
        'tosa;': '\u2929',
        'tprime;': '\u2034',
        'TRADE;': '\u2122',
        'trade;': '\u2122',
        'triangle;': '\u25b5',
        'triangledown;': '\u25bf',
        'triangleleft;': '\u25c3',
        'trianglelefteq;': '\u22b4',
        'triangleq;': '\u225c',
        'triangleright;': '\u25b9',
        'trianglerighteq;': '\u22b5',
        'tridot;': '\u25ec',
        'trie;': '\u225c',
        'triminus;': '\u2a3a',
        'TripleDot;': '\u20db',
        'triplus;': '\u2a39',
        'trisb;': '\u29cd',
        'tritime;': '\u2a3b',
        'trpezium;': '\u23e2',
        'Tscr;': '\U0001d4af',
        'tscr;': '\U0001d4c9',
        'TScy;': '\u0426',
        'tscy;': '\u0446',
        'TSHcy;': '\u040b',
        'tshcy;': '\u045b',
        'Tstrok;': '\u0166',
        'tstrok;': '\u0167',
        'twixt;': '\u226c',
        'twoheadleftarrow;': '\u219e',
        'twoheadrightarrow;': '\u21a0',
        'Uacute': '\xda',
        'uacute': '\xfa',
        'Uacute;': '\xda',
        'uacute;': '\xfa',
        'Uarr;': '\u219f',
        'uArr;': '\u21d1',
        'uarr;': '\u2191',
        'Uarrocir;': '\u2949',
        'Ubrcy;': '\u040e',
        'ubrcy;': '\u045e',
        'Ubreve;': '\u016c',
        'ubreve;': '\u016d',
        'Ucirc': '\xdb',
        'ucirc': '\xfb',
        'Ucirc;': '\xdb',
        'ucirc;': '\xfb',
        'Ucy;': '\u0423',
        'ucy;': '\u0443',
        'udarr;': '\u21c5',
        'Udblac;': '\u0170',
        'udblac;': '\u0171',
        'udhar;': '\u296e',
        'ufisht;': '\u297e',
        'Ufr;': '\U0001d518',
        'ufr;': '\U0001d532',
        'Ugrave': '\xd9',
        'ugrave': '\xf9',
        'Ugrave;': '\xd9',
        'ugrave;': '\xf9',
        'uHar;': '\u2963',
        'uharl;': '\u21bf',
        'uharr;': '\u21be',
        'uhblk;': '\u2580',
        'ulcorn;': '\u231c',
        'ulcorner;': '\u231c',
        'ulcrop;': '\u230f',
        'ultri;': '\u25f8',
        'Umacr;': '\u016a',
        'umacr;': '\u016b',
        'uml': '\xa8',
        'uml;': '\xa8',
        'UnderBar;': '_',
        'UnderBrace;': '\u23df',
        'UnderBracket;': '\u23b5',
        'UnderParenthesis;': '\u23dd',
        'Union;': '\u22c3',
        'UnionPlus;': '\u228e',
        'Uogon;': '\u0172',
        'uogon;': '\u0173',
        'Uopf;': '\U0001d54c',
        'uopf;': '\U0001d566',
        'UpArrow;': '\u2191',
        'Uparrow;': '\u21d1',
        'uparrow;': '\u2191',
        'UpArrowBar;': '\u2912',
        'UpArrowDownArrow;': '\u21c5',
        'UpDownArrow;': '\u2195',
        'Updownarrow;': '\u21d5',
        'updownarrow;': '\u2195',
        'UpEquilibrium;': '\u296e',
        'upharpoonleft;': '\u21bf',
        'upharpoonright;': '\u21be',
        'uplus;': '\u228e',
        'UpperLeftArrow;': '\u2196',
        'UpperRightArrow;': '\u2197',
        'Upsi;': '\u03d2',
        'upsi;': '\u03c5',
        'upsih;': '\u03d2',
        'Upsilon;': '\u03a5',
        'upsilon;': '\u03c5',
        'UpTee;': '\u22a5',
        'UpTeeArrow;': '\u21a5',
        'upuparrows;': '\u21c8',
        'urcorn;': '\u231d',
        'urcorner;': '\u231d',
        'urcrop;': '\u230e',
        'Uring;': '\u016e',
        'uring;': '\u016f',
        'urtri;': '\u25f9',
        'Uscr;': '\U0001d4b0',
        'uscr;': '\U0001d4ca',
        'utdot;': '\u22f0',
        'Utilde;': '\u0168',
        'utilde;': '\u0169',
        'utri;': '\u25b5',
        'utrif;': '\u25b4',
        'uuarr;': '\u21c8',
        'Uuml': '\xdc',
        'uuml': '\xfc',
        'Uuml;': '\xdc',
        'uuml;': '\xfc',
        'uwangle;': '\u29a7',
        'vangrt;': '\u299c',
        'varepsilon;': '\u03f5',
        'varkappa;': '\u03f0',
        'varnothing;': '\u2205',
        'varphi;': '\u03d5',
        'varpi;': '\u03d6',
        'varpropto;': '\u221d',
        'vArr;': '\u21d5',
        'varr;': '\u2195',
        'varrho;': '\u03f1',
        'varsigma;': '\u03c2',
        'varsubsetneq;': '\u228a\ufe00',
        'varsubsetneqq;': '\u2acb\ufe00',
        'varsupsetneq;': '\u228b\ufe00',
        'varsupsetneqq;': '\u2acc\ufe00',
        'vartheta;': '\u03d1',
        'vartriangleleft;': '\u22b2',
        'vartriangleright;': '\u22b3',
        'Vbar;': '\u2aeb',
        'vBar;': '\u2ae8',
        'vBarv;': '\u2ae9',
        'Vcy;': '\u0412',
        'vcy;': '\u0432',
        'VDash;': '\u22ab',
        'Vdash;': '\u22a9',
        'vDash;': '\u22a8',
        'vdash;': '\u22a2',
        'Vdashl;': '\u2ae6',
        'Vee;': '\u22c1',
        'vee;': '\u2228',
        'veebar;': '\u22bb',
        'veeeq;': '\u225a',
        'vellip;': '\u22ee',
        'Verbar;': '\u2016',
        'verbar;': '|',
        'Vert;': '\u2016',
        'vert;': '|',
        'VerticalBar;': '\u2223',
        'VerticalLine;': '|',
        'VerticalSeparator;': '\u2758',
        'VerticalTilde;': '\u2240',
        'VeryThinSpace;': '\u200a',
        'Vfr;': '\U0001d519',
        'vfr;': '\U0001d533',
        'vltri;': '\u22b2',
        'vnsub;': '\u2282\u20d2',
        'vnsup;': '\u2283\u20d2',
        'Vopf;': '\U0001d54d',
        'vopf;': '\U0001d567',
        'vprop;': '\u221d',
        'vrtri;': '\u22b3',
        'Vscr;': '\U0001d4b1',
        'vscr;': '\U0001d4cb',
        'vsubnE;': '\u2acb\ufe00',
        'vsubne;': '\u228a\ufe00',
        'vsupnE;': '\u2acc\ufe00',
        'vsupne;': '\u228b\ufe00',
        'Vvdash;': '\u22aa',
        'vzigzag;': '\u299a',
        'Wcirc;': '\u0174',
        'wcirc;': '\u0175',
        'wedbar;': '\u2a5f',
        'Wedge;': '\u22c0',
        'wedge;': '\u2227',
        'wedgeq;': '\u2259',
        'weierp;': '\u2118',
        'Wfr;': '\U0001d51a',
        'wfr;': '\U0001d534',
        'Wopf;': '\U0001d54e',
        'wopf;': '\U0001d568',
        'wp;': '\u2118',
        'wr;': '\u2240',
        'wreath;': '\u2240',
        'Wscr;': '\U0001d4b2',
        'wscr;': '\U0001d4cc',
        'xcap;': '\u22c2',
        'xcirc;': '\u25ef',
        'xcup;': '\u22c3',
        'xdtri;': '\u25bd',
        'Xfr;': '\U0001d51b',
        'xfr;': '\U0001d535',
        'xhArr;': '\u27fa',
        'xharr;': '\u27f7',
        'Xi;': '\u039e',
        'xi;': '\u03be',
        'xlArr;': '\u27f8',
        'xlarr;': '\u27f5',
        'xmap;': '\u27fc',
        'xnis;': '\u22fb',
        'xodot;': '\u2a00',
        'Xopf;': '\U0001d54f',
        'xopf;': '\U0001d569',
        'xoplus;': '\u2a01',
        'xotime;': '\u2a02',
        'xrArr;': '\u27f9',
        'xrarr;': '\u27f6',
        'Xscr;': '\U0001d4b3',
        'xscr;': '\U0001d4cd',
        'xsqcup;': '\u2a06',
        'xuplus;': '\u2a04',
        'xutri;': '\u25b3',
        'xvee;': '\u22c1',
        'xwedge;': '\u22c0',
        'Yacute': '\xdd',
        'yacute': '\xfd',
        'Yacute;': '\xdd',
        'yacute;': '\xfd',
        'YAcy;': '\u042f',
        'yacy;': '\u044f',
        'Ycirc;': '\u0176',
        'ycirc;': '\u0177',
        'Ycy;': '\u042b',
        'ycy;': '\u044b',
        'yen': '\xa5',
        'yen;': '\xa5',
        'Yfr;': '\U0001d51c',
        'yfr;': '\U0001d536',
        'YIcy;': '\u0407',
        'yicy;': '\u0457',
        'Yopf;': '\U0001d550',
        'yopf;': '\U0001d56a',
        'Yscr;': '\U0001d4b4',
        'yscr;': '\U0001d4ce',
        'YUcy;': '\u042e',
        'yucy;': '\u044e',
        'yuml': '\xff',
        'Yuml;': '\u0178',
        'yuml;': '\xff',
        'Zacute;': '\u0179',
        'zacute;': '\u017a',
        'Zcaron;': '\u017d',
        'zcaron;': '\u017e',
        'Zcy;': '\u0417',
        'zcy;': '\u0437',
        'Zdot;': '\u017b',
        'zdot;': '\u017c',
        'zeetrf;': '\u2128',
        'ZeroWidthSpace;': '\u200b',
        'Zeta;': '\u0396',
        'zeta;': '\u03b6',
        'Zfr;': '\u2128',
        'zfr;': '\U0001d537',
        'ZHcy;': '\u0416',
        'zhcy;': '\u0436',
        'zigrarr;': '\u21dd',
        'Zopf;': '\u2124',
        'zopf;': '\U0001d56b',
        'Zscr;': '\U0001d4b5',
        'zscr;': '\U0001d4cf',
        'zwj;': '\u200d',
        'zwnj;': '\u200c',
    }

try:
    import http.client as compat_http_client
except ImportError:  # Python 2
    import httplib as compat_http_client

try:
    from urllib.error import HTTPError as compat_HTTPError
except ImportError:  # Python 2
    from urllib2 import HTTPError as compat_HTTPError

try:
    from urllib.request import urlretrieve as compat_urlretrieve
except ImportError:  # Python 2
    from urllib import urlretrieve as compat_urlretrieve

try:
    from html.parser import HTMLParser as compat_HTMLParser
except ImportError:  # Python 2
    from HTMLParser import HTMLParser as compat_HTMLParser

try:  # Python 2
    from HTMLParser import HTMLParseError as compat_HTMLParseError
except ImportError:  # Python <3.4
    try:
        from html.parser import HTMLParseError as compat_HTMLParseError
    except ImportError:  # Python >3.4

        # HTMLParseError has been deprecated in Python 3.3 and removed in
        # Python 3.5. Introducing dummy exception for Python >3.5 for compatible
        # and uniform cross-version exceptiong handling
        class compat_HTMLParseError(Exception):
            pass

try:
    from subprocess import DEVNULL
    compat_subprocess_get_DEVNULL = lambda: DEVNULL
except ImportError:
    compat_subprocess_get_DEVNULL = lambda: open(os.path.devnull, 'w')

try:
    import http.server as compat_http_server
except ImportError:
    import BaseHTTPServer as compat_http_server

try:
    compat_str = unicode  # Python 2
except NameError:
    compat_str = str

try:
    from urllib.parse import unquote_to_bytes as compat_urllib_parse_unquote_to_bytes
    from urllib.parse import unquote as compat_urllib_parse_unquote
    from urllib.parse import unquote_plus as compat_urllib_parse_unquote_plus
except ImportError:  # Python 2
    _asciire = (compat_urllib_parse._asciire if hasattr(compat_urllib_parse, '_asciire')
                else re.compile(r'([\x00-\x7f]+)'))

    # HACK: The following are the correct unquote_to_bytes, unquote and unquote_plus
    # implementations from cpython 3.4.3's stdlib. Python 2's version
    # is apparently broken (see https://github.com/ytdl-org/youtube-dl/pull/6244)

    def compat_urllib_parse_unquote_to_bytes(string):
        """unquote_to_bytes('abc%20def') -> b'abc def'."""
        # Note: strings are encoded as UTF-8. This is only an issue if it contains
        # unescaped non-ASCII characters, which URIs should not.
        if not string:
            # Is it a string-like object?
            string.split
            return b''
        if isinstance(string, compat_str):
            string = string.encode('utf-8')
        bits = string.split(b'%')
        if len(bits) == 1:
            return string
        res = [bits[0]]
        append = res.append
        for item in bits[1:]:
            try:
                append(compat_urllib_parse._hextochr[item[:2]])
                append(item[2:])
            except KeyError:
                append(b'%')
                append(item)
        return b''.join(res)

    def compat_urllib_parse_unquote(string, encoding='utf-8', errors='replace'):
        """Replace %xx escapes by their single-character equivalent. The optional
        encoding and errors parameters specify how to decode percent-encoded
        sequences into Unicode characters, as accepted by the bytes.decode()
        method.
        By default, percent-encoded sequences are decoded with UTF-8, and invalid
        sequences are replaced by a placeholder character.

        unquote('abc%20def') -> 'abc def'.
        """
        if '%' not in string:
            string.split
            return string
        if encoding is None:
            encoding = 'utf-8'
        if errors is None:
            errors = 'replace'
        bits = _asciire.split(string)
        res = [bits[0]]
        append = res.append
        for i in range(1, len(bits), 2):
            append(compat_urllib_parse_unquote_to_bytes(bits[i]).decode(encoding, errors))
            append(bits[i + 1])
        return ''.join(res)

    def compat_urllib_parse_unquote_plus(string, encoding='utf-8', errors='replace'):
        """Like unquote(), but also replace plus signs by spaces, as required for
        unquoting HTML form values.

        unquote_plus('%7e/abc+def') -> '~/abc def'
        """
        string = string.replace('+', ' ')
        return compat_urllib_parse_unquote(string, encoding, errors)

try:
    from urllib.parse import urlencode as compat_urllib_parse_urlencode
except ImportError:  # Python 2
    # Python 2 will choke in urlencode on mixture of byte and unicode strings.
    # Possible solutions are to either port it from python 3 with all
    # the friends or manually ensure input query contains only byte strings.
    # We will stick with latter thus recursively encoding the whole query.
    def compat_urllib_parse_urlencode(query, doseq=0, encoding='utf-8'):
        def encode_elem(e):
            if isinstance(e, dict):
                e = encode_dict(e)
            elif isinstance(e, (list, tuple,)):
                list_e = encode_list(e)
                e = tuple(list_e) if isinstance(e, tuple) else list_e
            elif isinstance(e, compat_str):
                e = e.encode(encoding)
            return e

        def encode_dict(d):
            return dict((encode_elem(k), encode_elem(v)) for k, v in d.items())

        def encode_list(l):
            return [encode_elem(e) for e in l]

        return compat_urllib_parse.urlencode(encode_elem(query), doseq=doseq)

try:
    from urllib.request import DataHandler as compat_urllib_request_DataHandler
except ImportError:  # Python < 3.4
    # Ported from CPython 98774:1733b3bd46db, Lib/urllib/request.py
    class compat_urllib_request_DataHandler(compat_urllib_request.BaseHandler):
        def data_open(self, req):
            # data URLs as specified in RFC 2397.
            #
            # ignores POSTed data
            #
            # syntax:
            # dataurl   := "data:" [ mediatype ] [ ";base64" ] "," data
            # mediatype := [ type "/" subtype ] *( ";" parameter )
            # data      := *urlchar
            # parameter := attribute "=" value
            url = req.get_full_url()

            scheme, data = url.split(':', 1)
            mediatype, data = data.split(',', 1)

            # even base64 encoded data URLs might be quoted so unquote in any case:
            data = compat_urllib_parse_unquote_to_bytes(data)
            if mediatype.endswith(';base64'):
                data = binascii.a2b_base64(data)
                mediatype = mediatype[:-7]

            if not mediatype:
                mediatype = 'text/plain;charset=US-ASCII'

            headers = email.message_from_string(
                'Content-type: %s\nContent-length: %d\n' % (mediatype, len(data)))

            return compat_urllib_response.addinfourl(io.BytesIO(data), headers, url)

try:
    compat_basestring = basestring  # Python 2
except NameError:
    compat_basestring = str

try:
    compat_chr = unichr  # Python 2
except NameError:
    compat_chr = chr

try:
    from xml.etree.ElementTree import ParseError as compat_xml_parse_error
except ImportError:  # Python 2.6
    from xml.parsers.expat import ExpatError as compat_xml_parse_error


etree = xml.etree.ElementTree


class _TreeBuilder(etree.TreeBuilder):
    def doctype(self, name, pubid, system):
        pass


try:
    # xml.etree.ElementTree.Element is a method in Python <=2.6 and
    # the following will crash with:
    #  TypeError: isinstance() arg 2 must be a class, type, or tuple of classes and types
    isinstance(None, xml.etree.ElementTree.Element)
    from xml.etree.ElementTree import Element as compat_etree_Element
except TypeError:  # Python <=2.6
    from xml.etree.ElementTree import _ElementInterface as compat_etree_Element

if sys.version_info[0] >= 3:
    def compat_etree_fromstring(text):
        return etree.XML(text, parser=etree.XMLParser(target=_TreeBuilder()))
else:
    # python 2.x tries to encode unicode strings with ascii (see the
    # XMLParser._fixtext method)
    try:
        _etree_iter = etree.Element.iter
    except AttributeError:  # Python <=2.6
        def _etree_iter(root):
            for el in root.findall('*'):
                yield el
                for sub in _etree_iter(el):
                    yield sub

    # on 2.6 XML doesn't have a parser argument, function copied from CPython
    # 2.7 source
    def _XML(text, parser=None):
        if not parser:
            parser = etree.XMLParser(target=_TreeBuilder())
        parser.feed(text)
        return parser.close()

    def _element_factory(*args, **kwargs):
        el = etree.Element(*args, **kwargs)
        for k, v in el.items():
            if isinstance(v, bytes):
                el.set(k, v.decode('utf-8'))
        return el

    def compat_etree_fromstring(text):
        doc = _XML(text, parser=etree.XMLParser(target=_TreeBuilder(element_factory=_element_factory)))
        for el in _etree_iter(doc):
            if el.text is not None and isinstance(el.text, bytes):
                el.text = el.text.decode('utf-8')
        return doc

if hasattr(etree, 'register_namespace'):
    compat_etree_register_namespace = etree.register_namespace
else:
    def compat_etree_register_namespace(prefix, uri):
        """Register a namespace prefix.
        The registry is global, and any existing mapping for either the
        given prefix or the namespace URI will be removed.
        *prefix* is the namespace prefix, *uri* is a namespace uri. Tags and
        attributes in this namespace will be serialized with prefix if possible.
        ValueError is raised if prefix is reserved or is invalid.
        """
        if re.match(r"ns\d+$", prefix):
            raise ValueError("Prefix format reserved for internal use")
        for k, v in list(etree._namespace_map.items()):
            if k == uri or v == prefix:
                del etree._namespace_map[k]
        etree._namespace_map[uri] = prefix

if sys.version_info < (2, 7):
    # Here comes the crazy part: In 2.6, if the xpath is a unicode,
    # .//node does not match if a node is a direct child of . !
    def compat_xpath(xpath):
        if isinstance(xpath, compat_str):
            xpath = xpath.encode('ascii')
        return xpath
else:
    compat_xpath = lambda xpath: xpath

try:
    from urllib.parse import parse_qs as compat_parse_qs
except ImportError:  # Python 2
    # HACK: The following is the correct parse_qs implementation from cpython 3's stdlib.
    # Python 2's version is apparently totally broken

    def _parse_qsl(qs, keep_blank_values=False, strict_parsing=False,
                   encoding='utf-8', errors='replace'):
        qs, _coerce_result = qs, compat_str
        pairs = [s2 for s1 in qs.split('&') for s2 in s1.split(';')]
        r = []
        for name_value in pairs:
            if not name_value and not strict_parsing:
                continue
            nv = name_value.split('=', 1)
            if len(nv) != 2:
                if strict_parsing:
                    raise ValueError('bad query field: %r' % (name_value,))
                # Handle case of a control-name with no equal sign
                if keep_blank_values:
                    nv.append('')
                else:
                    continue
            if len(nv[1]) or keep_blank_values:
                name = nv[0].replace('+', ' ')
                name = compat_urllib_parse_unquote(
                    name, encoding=encoding, errors=errors)
                name = _coerce_result(name)
                value = nv[1].replace('+', ' ')
                value = compat_urllib_parse_unquote(
                    value, encoding=encoding, errors=errors)
                value = _coerce_result(value)
                r.append((name, value))
        return r

    def compat_parse_qs(qs, keep_blank_values=False, strict_parsing=False,
                        encoding='utf-8', errors='replace'):
        parsed_result = {}
        pairs = _parse_qsl(qs, keep_blank_values, strict_parsing,
                           encoding=encoding, errors=errors)
        for name, value in pairs:
            if name in parsed_result:
                parsed_result[name].append(value)
            else:
                parsed_result[name] = [value]
        return parsed_result


compat_os_name = os._name if os.name == 'java' else os.name


if compat_os_name == 'nt':
    def compat_shlex_quote(s):
        return s if re.match(r'^[-_\w./]+$', s) else '"%s"' % s.replace('"', '\\"')
else:
    try:
        from shlex import quote as compat_shlex_quote
    except ImportError:  # Python < 3.3
        def compat_shlex_quote(s):
            if re.match(r'^[-_\w./]+$', s):
                return s
            else:
                return "'" + s.replace("'", "'\"'\"'") + "'"


try:
    args = shlex.split('中文')
    assert (isinstance(args, list)
            and isinstance(args[0], compat_str)
            and args[0] == '中文')
    compat_shlex_split = shlex.split
except (AssertionError, UnicodeEncodeError):
    # Working around shlex issue with unicode strings on some python 2
    # versions (see http://bugs.python.org/issue1548891)
    def compat_shlex_split(s, comments=False, posix=True):
        if isinstance(s, compat_str):
            s = s.encode('utf-8')
        return list(map(lambda s: s.decode('utf-8'), shlex.split(s, comments, posix)))


def compat_ord(c):
    if type(c) is int:
        return c
    else:
        return ord(c)


if sys.version_info >= (3, 0):
    compat_getenv = os.getenv
    compat_expanduser = os.path.expanduser

    def compat_setenv(key, value, env=os.environ):
        env[key] = value
else:
    # Environment variables should be decoded with filesystem encoding.
    # Otherwise it will fail if any non-ASCII characters present (see #3854 #3217 #2918)

    def compat_getenv(key, default=None):
        from .utils import get_filesystem_encoding
        env = os.getenv(key, default)
        if env:
            env = env.decode(get_filesystem_encoding())
        return env

    def compat_setenv(key, value, env=os.environ):
        def encode(v):
            from .utils import get_filesystem_encoding
            return v.encode(get_filesystem_encoding()) if isinstance(v, compat_str) else v
        env[encode(key)] = encode(value)

    # HACK: The default implementations of os.path.expanduser from cpython do not decode
    # environment variables with filesystem encoding. We will work around this by
    # providing adjusted implementations.
    # The following are os.path.expanduser implementations from cpython 2.7.8 stdlib
    # for different platforms with correct environment variables decoding.

    if compat_os_name == 'posix':
        def compat_expanduser(path):
            """Expand ~ and ~user constructions.  If user or $HOME is unknown,
            do nothing."""
            if not path.startswith('~'):
                return path
            i = path.find('/', 1)
            if i < 0:
                i = len(path)
            if i == 1:
                if 'HOME' not in os.environ:
                    import pwd
                    userhome = pwd.getpwuid(os.getuid()).pw_dir
                else:
                    userhome = compat_getenv('HOME')
            else:
                import pwd
                try:
                    pwent = pwd.getpwnam(path[1:i])
                except KeyError:
                    return path
                userhome = pwent.pw_dir
            userhome = userhome.rstrip('/')
            return (userhome + path[i:]) or '/'
    elif compat_os_name in ('nt', 'ce'):
        def compat_expanduser(path):
            """Expand ~ and ~user constructs.

            If user or $HOME is unknown, do nothing."""
            if path[:1] != '~':
                return path
            i, n = 1, len(path)
            while i < n and path[i] not in '/\\':
                i = i + 1

            if 'HOME' in os.environ:
                userhome = compat_getenv('HOME')
            elif 'USERPROFILE' in os.environ:
                userhome = compat_getenv('USERPROFILE')
            elif 'HOMEPATH' not in os.environ:
                return path
            else:
                try:
                    drive = compat_getenv('HOMEDRIVE')
                except KeyError:
                    drive = ''
                userhome = os.path.join(drive, compat_getenv('HOMEPATH'))

            if i != 1:  # ~user
                userhome = os.path.join(os.path.dirname(userhome), path[1:i])

            return userhome + path[i:]
    else:
        compat_expanduser = os.path.expanduser


if compat_os_name == 'nt' and sys.version_info < (3, 8):
    # os.path.realpath on Windows does not follow symbolic links
    # prior to Python 3.8 (see https://bugs.python.org/issue9949)
    def compat_realpath(path):
        while os.path.islink(path):
            path = os.path.abspath(os.readlink(path))
        return path
else:
    compat_realpath = os.path.realpath


if sys.version_info < (3, 0):
    def compat_print(s):
        from .utils import preferredencoding
        print(s.encode(preferredencoding(), 'xmlcharrefreplace'))
else:
    def compat_print(s):
        assert isinstance(s, compat_str)
        print(s)


if sys.version_info < (3, 0) and sys.platform == 'win32':
    def compat_getpass(prompt, *args, **kwargs):
        if isinstance(prompt, compat_str):
            from .utils import preferredencoding
            prompt = prompt.encode(preferredencoding())
        return getpass.getpass(prompt, *args, **kwargs)
else:
    compat_getpass = getpass.getpass

try:
    compat_input = raw_input
except NameError:  # Python 3
    compat_input = input

# Python < 2.6.5 require kwargs to be bytes
try:
    def _testfunc(x):
        pass
    _testfunc(**{'x': 0})
except TypeError:
    def compat_kwargs(kwargs):
        return dict((bytes(k), v) for k, v in kwargs.items())
else:
    compat_kwargs = lambda kwargs: kwargs


try:
    compat_numeric_types = (int, float, long, complex)
except NameError:  # Python 3
    compat_numeric_types = (int, float, complex)


try:
    compat_integer_types = (int, long)
except NameError:  # Python 3
    compat_integer_types = (int, )


if sys.version_info < (2, 7):
    def compat_socket_create_connection(address, timeout, source_address=None):
        host, port = address
        err = None
        for res in socket.getaddrinfo(host, port, 0, socket.SOCK_STREAM):
            af, socktype, proto, canonname, sa = res
            sock = None
            try:
                sock = socket.socket(af, socktype, proto)
                sock.settimeout(timeout)
                if source_address:
                    sock.bind(source_address)
                sock.connect(sa)
                return sock
            except socket.error as _:
                err = _
                if sock is not None:
                    sock.close()
        if err is not None:
            raise err
        else:
            raise socket.error('getaddrinfo returns an empty list')
else:
    compat_socket_create_connection = socket.create_connection


# Fix https://github.com/ytdl-org/youtube-dl/issues/4223
# See http://bugs.python.org/issue9161 for what is broken
def workaround_optparse_bug9161():
    op = optparse.OptionParser()
    og = optparse.OptionGroup(op, 'foo')
    try:
        og.add_option('-t')
    except TypeError:
        real_add_option = optparse.OptionGroup.add_option

        def _compat_add_option(self, *args, **kwargs):
            enc = lambda v: (
                v.encode('ascii', 'replace') if isinstance(v, compat_str)
                else v)
            bargs = [enc(a) for a in args]
            bkwargs = dict(
                (k, enc(v)) for k, v in kwargs.items())
            return real_add_option(self, *bargs, **bkwargs)
        optparse.OptionGroup.add_option = _compat_add_option


if hasattr(shutil, 'get_terminal_size'):  # Python >= 3.3
    compat_get_terminal_size = shutil.get_terminal_size
else:
    _terminal_size = collections.namedtuple('terminal_size', ['columns', 'lines'])

    def compat_get_terminal_size(fallback=(80, 24)):
        columns = compat_getenv('COLUMNS')
        if columns:
            columns = int(columns)
        else:
            columns = None
        lines = compat_getenv('LINES')
        if lines:
            lines = int(lines)
        else:
            lines = None

        if columns is None or lines is None or columns <= 0 or lines <= 0:
            try:
                sp = subprocess.Popen(
                    ['stty', 'size'],
                    stdout=subprocess.PIPE, stderr=subprocess.PIPE)
                out, err = sp.communicate()
                _lines, _columns = map(int, out.split())
            except Exception:
                _columns, _lines = _terminal_size(*fallback)

            if columns is None or columns <= 0:
                columns = _columns
            if lines is None or lines <= 0:
                lines = _lines
        return _terminal_size(columns, lines)

try:
    itertools.count(start=0, step=1)
    compat_itertools_count = itertools.count
except TypeError:  # Python 2.6
    def compat_itertools_count(start=0, step=1):
        n = start
        while True:
            yield n
            n += step

if sys.version_info >= (3, 0):
    from tokenize import tokenize as compat_tokenize_tokenize
else:
    from tokenize import generate_tokens as compat_tokenize_tokenize


try:
    struct.pack('!I', 0)
except TypeError:
    # In Python 2.6 and 2.7.x < 2.7.7, struct requires a bytes argument
    # See https://bugs.python.org/issue19099
    def compat_struct_pack(spec, *args):
        if isinstance(spec, compat_str):
            spec = spec.encode('ascii')
        return struct.pack(spec, *args)

    def compat_struct_unpack(spec, *args):
        if isinstance(spec, compat_str):
            spec = spec.encode('ascii')
        return struct.unpack(spec, *args)

    class compat_Struct(struct.Struct):
        def __init__(self, fmt):
            if isinstance(fmt, compat_str):
                fmt = fmt.encode('ascii')
            super(compat_Struct, self).__init__(fmt)
else:
    compat_struct_pack = struct.pack
    compat_struct_unpack = struct.unpack
    if platform.python_implementation() == 'IronPython' and sys.version_info < (2, 7, 8):
        class compat_Struct(struct.Struct):
            def unpack(self, string):
                if not isinstance(string, buffer):  # noqa: F821
                    string = buffer(string)  # noqa: F821
                return super(compat_Struct, self).unpack(string)
    else:
        compat_Struct = struct.Struct


try:
    from future_builtins import zip as compat_zip
except ImportError:  # not 2.6+ or is 3.x
    try:
        from itertools import izip as compat_zip  # < 2.5 or 3.x
    except ImportError:
        compat_zip = zip


if sys.version_info < (3, 3):
    def compat_b64decode(s, *args, **kwargs):
        if isinstance(s, compat_str):
            s = s.encode('ascii')
        return base64.b64decode(s, *args, **kwargs)
else:
    compat_b64decode = base64.b64decode


if platform.python_implementation() == 'PyPy' and sys.pypy_version_info < (5, 4, 0):
    # PyPy2 prior to version 5.4.0 expects byte strings as Windows function
    # names, see the original PyPy issue [1] and the youtube-dl one [2].
    # 1. https://bitbucket.org/pypy/pypy/issues/2360/windows-ctypescdll-typeerror-function-name
    # 2. https://github.com/ytdl-org/youtube-dl/pull/4392
    def compat_ctypes_WINFUNCTYPE(*args, **kwargs):
        real = ctypes.WINFUNCTYPE(*args, **kwargs)

        def resf(tpl, *args, **kwargs):
            funcname, dll = tpl
            return real((str(funcname), dll), *args, **kwargs)

        return resf
else:
    def compat_ctypes_WINFUNCTYPE(*args, **kwargs):
        return ctypes.WINFUNCTYPE(*args, **kwargs)


__all__ = [
    'compat_HTMLParseError',
    'compat_HTMLParser',
    'compat_HTTPError',
    'compat_Struct',
    'compat_b64decode',
    'compat_basestring',
    'compat_chr',
    'compat_cookiejar',
    'compat_cookiejar_Cookie',
    'compat_cookies',
    'compat_ctypes_WINFUNCTYPE',
    'compat_etree_Element',
    'compat_etree_fromstring',
    'compat_etree_register_namespace',
    'compat_expanduser',
    'compat_get_terminal_size',
    'compat_getenv',
    'compat_getpass',
    'compat_html_entities',
    'compat_html_entities_html5',
    'compat_http_client',
    'compat_http_server',
    'compat_input',
    'compat_integer_types',
    'compat_itertools_count',
    'compat_kwargs',
    'compat_numeric_types',
    'compat_ord',
    'compat_os_name',
    'compat_parse_qs',
    'compat_print',
    'compat_realpath',
    'compat_setenv',
    'compat_shlex_quote',
    'compat_shlex_split',
    'compat_socket_create_connection',
    'compat_str',
    'compat_struct_pack',
    'compat_struct_unpack',
    'compat_subprocess_get_DEVNULL',
    'compat_tokenize_tokenize',
    'compat_urllib_error',
    'compat_urllib_parse',
    'compat_urllib_parse_unquote',
    'compat_urllib_parse_unquote_plus',
    'compat_urllib_parse_unquote_to_bytes',
    'compat_urllib_parse_urlencode',
    'compat_urllib_parse_urlparse',
    'compat_urllib_request',
    'compat_urllib_request_DataHandler',
    'compat_urllib_response',
    'compat_urlparse',
    'compat_urlretrieve',
    'compat_xml_parse_error',
    'compat_xpath',
    'compat_zip',
    'workaround_optparse_bug9161',
]
#!/usr/bin/env python
# coding: utf-8

from __future__ import unicode_literals

__license__ = 'Public Domain'

import codecs
import io
import os
import random
import sys


from .options import (
    parseOpts,
)
from .compat import (
    compat_getpass,
    compat_shlex_split,
    workaround_optparse_bug9161,
)
from .utils import (
    DateRange,
    decodeOption,
    DEFAULT_OUTTMPL,
    DownloadError,
    expand_path,
    match_filter_func,
    MaxDownloadsReached,
    preferredencoding,
    read_batch_urls,
    SameFileError,
    setproctitle,
    std_headers,
    write_string,
    render_table,
)
from .update import update_self
from .downloader import (
    FileDownloader,
)
from .extractor import gen_extractors, list_extractors
from .extractor.adobepass import MSO_INFO
from .YoutubeDL import YoutubeDL


def _real_main(argv=None):
    # Compatibility fixes for Windows
    if sys.platform == 'win32':
        # https://github.com/ytdl-org/youtube-dl/issues/820
        codecs.register(lambda name: codecs.lookup('utf-8') if name == 'cp65001' else None)

    workaround_optparse_bug9161()

    setproctitle('youtube-dl')

    parser, opts, args = parseOpts(argv)

    # Set user agent
    if opts.user_agent is not None:
        std_headers['User-Agent'] = opts.user_agent

    # Set referer
    if opts.referer is not None:
        std_headers['Referer'] = opts.referer

    # Custom HTTP headers
    if opts.headers is not None:
        for h in opts.headers:
            if ':' not in h:
                parser.error('wrong header formatting, it should be key:value, not "%s"' % h)
            key, value = h.split(':', 1)
            if opts.verbose:
                write_string('[debug] Adding header from command line option %s:%s\n' % (key, value))
            std_headers[key] = value

    # Dump user agent
    if opts.dump_user_agent:
        write_string(std_headers['User-Agent'] + '\n', out=sys.stdout)
        sys.exit(0)

    # Batch file verification
    batch_urls = []
    if opts.batchfile is not None:
        try:
            if opts.batchfile == '-':
                batchfd = sys.stdin
            else:
                batchfd = io.open(
                    expand_path(opts.batchfile),
                    'r', encoding='utf-8', errors='ignore')
            batch_urls = read_batch_urls(batchfd)
            if opts.verbose:
                write_string('[debug] Batch file urls: ' + repr(batch_urls) + '\n')
        except IOError:
            sys.exit('ERROR: batch file %s could not be read' % opts.batchfile)
    all_urls = batch_urls + [url.strip() for url in args]  # batch_urls are already striped in read_batch_urls
    _enc = preferredencoding()
    all_urls = [url.decode(_enc, 'ignore') if isinstance(url, bytes) else url for url in all_urls]

    if opts.list_extractors:
        for ie in list_extractors(opts.age_limit):
            write_string(ie.IE_NAME + (' (CURRENTLY BROKEN)' if not ie._WORKING else '') + '\n', out=sys.stdout)
            matchedUrls = [url for url in all_urls if ie.suitable(url)]
            for mu in matchedUrls:
                write_string('  ' + mu + '\n', out=sys.stdout)
        sys.exit(0)
    if opts.list_extractor_descriptions:
        for ie in list_extractors(opts.age_limit):
            if not ie._WORKING:
                continue
            desc = getattr(ie, 'IE_DESC', ie.IE_NAME)
            if desc is False:
                continue
            if hasattr(ie, 'SEARCH_KEY'):
                _SEARCHES = ('cute kittens', 'slithering pythons', 'falling cat', 'angry poodle', 'purple fish', 'running tortoise', 'sleeping bunny', 'burping cow')
                _COUNTS = ('', '5', '10', 'all')
                desc += ' (Example: "%s%s:%s" )' % (ie.SEARCH_KEY, random.choice(_COUNTS), random.choice(_SEARCHES))
            write_string(desc + '\n', out=sys.stdout)
        sys.exit(0)
    if opts.ap_list_mso:
        table = [[mso_id, mso_info['name']] for mso_id, mso_info in MSO_INFO.items()]
        write_string('Supported TV Providers:\n' + render_table(['mso', 'mso name'], table) + '\n', out=sys.stdout)
        sys.exit(0)

    # Conflicting, missing and erroneous options
    if opts.usenetrc and (opts.username is not None or opts.password is not None):
        parser.error('using .netrc conflicts with giving username/password')
    if opts.password is not None and opts.username is None:
        parser.error('account username missing\n')
    if opts.ap_password is not None and opts.ap_username is None:
        parser.error('TV Provider account username missing\n')
    if opts.outtmpl is not None and (opts.usetitle or opts.autonumber or opts.useid):
        parser.error('using output template conflicts with using title, video ID or auto number')
    if opts.autonumber_size is not None:
        if opts.autonumber_size <= 0:
            parser.error('auto number size must be positive')
    if opts.autonumber_start is not None:
        if opts.autonumber_start < 0:
            parser.error('auto number start must be positive or 0')
    if opts.usetitle and opts.useid:
        parser.error('using title conflicts with using video ID')
    if opts.username is not None and opts.password is None:
        opts.password = compat_getpass('Type account password and press [Return]: ')
    if opts.ap_username is not None and opts.ap_password is None:
        opts.ap_password = compat_getpass('Type TV provider account password and press [Return]: ')
    if opts.ratelimit is not None:
        numeric_limit = FileDownloader.parse_bytes(opts.ratelimit)
        if numeric_limit is None:
            parser.error('invalid rate limit specified')
        opts.ratelimit = numeric_limit
    if opts.min_filesize is not None:
        numeric_limit = FileDownloader.parse_bytes(opts.min_filesize)
        if numeric_limit is None:
            parser.error('invalid min_filesize specified')
        opts.min_filesize = numeric_limit
    if opts.max_filesize is not None:
        numeric_limit = FileDownloader.parse_bytes(opts.max_filesize)
        if numeric_limit is None:
            parser.error('invalid max_filesize specified')
        opts.max_filesize = numeric_limit
    if opts.sleep_interval is not None:
        if opts.sleep_interval < 0:
            parser.error('sleep interval must be positive or 0')
    if opts.max_sleep_interval is not None:
        if opts.max_sleep_interval < 0:
            parser.error('max sleep interval must be positive or 0')
        if opts.sleep_interval is None:
            parser.error('min sleep interval must be specified, use --min-sleep-interval')
        if opts.max_sleep_interval < opts.sleep_interval:
            parser.error('max sleep interval must be greater than or equal to min sleep interval')
    else:
        opts.max_sleep_interval = opts.sleep_interval
    if opts.ap_mso and opts.ap_mso not in MSO_INFO:
        parser.error('Unsupported TV Provider, use --ap-list-mso to get a list of supported TV Providers')

    def parse_retries(retries):
        if retries in ('inf', 'infinite'):
            parsed_retries = float('inf')
        else:
            try:
                parsed_retries = int(retries)
            except (TypeError, ValueError):
                parser.error('invalid retry count specified')
        return parsed_retries
    if opts.retries is not None:
        opts.retries = parse_retries(opts.retries)
    if opts.fragment_retries is not None:
        opts.fragment_retries = parse_retries(opts.fragment_retries)
    if opts.buffersize is not None:
        numeric_buffersize = FileDownloader.parse_bytes(opts.buffersize)
        if numeric_buffersize is None:
            parser.error('invalid buffer size specified')
        opts.buffersize = numeric_buffersize
    if opts.http_chunk_size is not None:
        numeric_chunksize = FileDownloader.parse_bytes(opts.http_chunk_size)
        if not numeric_chunksize:
            parser.error('invalid http chunk size specified')
        opts.http_chunk_size = numeric_chunksize
    if opts.playliststart <= 0:
        raise ValueError('Playlist start must be positive')
    if opts.playlistend not in (-1, None) and opts.playlistend < opts.playliststart:
        raise ValueError('Playlist end must be greater than playlist start')
    if opts.extractaudio:
        if opts.audioformat not in ['best', 'aac', 'flac', 'mp3', 'm4a', 'opus', 'vorbis', 'wav']:
            parser.error('invalid audio format specified')
    if opts.audioquality:
        opts.audioquality = opts.audioquality.strip('k').strip('K')
        if not opts.audioquality.isdigit():
            parser.error('invalid audio quality specified')
    if opts.recodevideo is not None:
        if opts.recodevideo not in ['mp4', 'flv', 'webm', 'ogg', 'mkv', 'avi']:
            parser.error('invalid video recode format specified')
    if opts.convertsubtitles is not None:
        if opts.convertsubtitles not in ['srt', 'vtt', 'ass', 'lrc']:
            parser.error('invalid subtitle format specified')

    if opts.date is not None:
        date = DateRange.day(opts.date)
    else:
        date = DateRange(opts.dateafter, opts.datebefore)

    # Do not download videos when there are audio-only formats
    if opts.extractaudio and not opts.keepvideo and opts.format is None:
        opts.format = 'bestaudio/best'

    # --all-sub automatically sets --write-sub if --write-auto-sub is not given
    # this was the old behaviour if only --all-sub was given.
    if opts.allsubtitles and not opts.writeautomaticsub:
        opts.writesubtitles = True

    outtmpl = ((opts.outtmpl is not None and opts.outtmpl)
               or (opts.format == '-1' and opts.usetitle and '%(title)s-%(id)s-%(format)s.%(ext)s')
               or (opts.format == '-1' and '%(id)s-%(format)s.%(ext)s')
               or (opts.usetitle and opts.autonumber and '%(autonumber)s-%(title)s-%(id)s.%(ext)s')
               or (opts.usetitle and '%(title)s-%(id)s.%(ext)s')
               or (opts.useid and '%(id)s.%(ext)s')
               or (opts.autonumber and '%(autonumber)s-%(id)s.%(ext)s')
               or DEFAULT_OUTTMPL)
    if not os.path.splitext(outtmpl)[1] and opts.extractaudio:
        parser.error('Cannot download a video and extract audio into the same'
                     ' file! Use "{0}.%(ext)s" instead of "{0}" as the output'
                     ' template'.format(outtmpl))

    any_getting = opts.geturl or opts.gettitle or opts.getid or opts.getthumbnail or opts.getdescription or opts.getfilename or opts.getformat or opts.getduration or opts.dumpjson or opts.dump_single_json
    any_printing = opts.print_json
    download_archive_fn = expand_path(opts.download_archive) if opts.download_archive is not None else opts.download_archive

    # PostProcessors
    postprocessors = []
    if opts.metafromtitle:
        postprocessors.append({
            'key': 'MetadataFromTitle',
            'titleformat': opts.metafromtitle
        })
    if opts.extractaudio:
        postprocessors.append({
            'key': 'FFmpegExtractAudio',
            'preferredcodec': opts.audioformat,
            'preferredquality': opts.audioquality,
            'nopostoverwrites': opts.nopostoverwrites,
        })
    if opts.recodevideo:
        postprocessors.append({
            'key': 'FFmpegVideoConvertor',
            'preferedformat': opts.recodevideo,
        })
    # FFmpegMetadataPP should be run after FFmpegVideoConvertorPP and
    # FFmpegExtractAudioPP as containers before conversion may not support
    # metadata (3gp, webm, etc.)
    # And this post-processor should be placed before other metadata
    # manipulating post-processors (FFmpegEmbedSubtitle) to prevent loss of
    # extra metadata. By default ffmpeg preserves metadata applicable for both
    # source and target containers. From this point the container won't change,
    # so metadata can be added here.
    if opts.addmetadata:
        postprocessors.append({'key': 'FFmpegMetadata'})
    if opts.convertsubtitles:
        postprocessors.append({
            'key': 'FFmpegSubtitlesConvertor',
            'format': opts.convertsubtitles,
        })
    if opts.embedsubtitles:
        postprocessors.append({
            'key': 'FFmpegEmbedSubtitle',
        })
    if opts.embedthumbnail:
        already_have_thumbnail = opts.writethumbnail or opts.write_all_thumbnails
        postprocessors.append({
            'key': 'EmbedThumbnail',
            'already_have_thumbnail': already_have_thumbnail
        })
        if not already_have_thumbnail:
            opts.writethumbnail = True
    # XAttrMetadataPP should be run after post-processors that may change file
    # contents
    if opts.xattrs:
        postprocessors.append({'key': 'XAttrMetadata'})
    # Please keep ExecAfterDownload towards the bottom as it allows the user to modify the final file in any way.
    # So if the user is able to remove the file before your postprocessor runs it might cause a few problems.
    if opts.exec_cmd:
        postprocessors.append({
            'key': 'ExecAfterDownload',
            'exec_cmd': opts.exec_cmd,
        })
    external_downloader_args = None
    if opts.external_downloader_args:
        external_downloader_args = compat_shlex_split(opts.external_downloader_args)
    postprocessor_args = None
    if opts.postprocessor_args:
        postprocessor_args = compat_shlex_split(opts.postprocessor_args)
    match_filter = (
        None if opts.match_filter is None
        else match_filter_func(opts.match_filter))

    ydl_opts = {
        'usenetrc': opts.usenetrc,
        'username': opts.username,
        'password': opts.password,
        'twofactor': opts.twofactor,
        'videopassword': opts.videopassword,
        'ap_mso': opts.ap_mso,
        'ap_username': opts.ap_username,
        'ap_password': opts.ap_password,
        'quiet': (opts.quiet or any_getting or any_printing),
        'no_warnings': opts.no_warnings,
        'forceurl': opts.geturl,
        'forcetitle': opts.gettitle,
        'forceid': opts.getid,
        'forcethumbnail': opts.getthumbnail,
        'forcedescription': opts.getdescription,
        'forceduration': opts.getduration,
        'forcefilename': opts.getfilename,
        'forceformat': opts.getformat,
        'forcejson': opts.dumpjson or opts.print_json,
        'dump_single_json': opts.dump_single_json,
        'simulate': opts.simulate or any_getting,
        'skip_download': opts.skip_download,
        'format': opts.format,
        'listformats': opts.listformats,
        'outtmpl': outtmpl,
        'autonumber_size': opts.autonumber_size,
        'autonumber_start': opts.autonumber_start,
        'restrictfilenames': opts.restrictfilenames,
        'ignoreerrors': opts.ignoreerrors,
        'force_generic_extractor': opts.force_generic_extractor,
        'ratelimit': opts.ratelimit,
        'nooverwrites': opts.nooverwrites,
        'retries': opts.retries,
        'fragment_retries': opts.fragment_retries,
        'skip_unavailable_fragments': opts.skip_unavailable_fragments,
        'keep_fragments': opts.keep_fragments,
        'buffersize': opts.buffersize,
        'noresizebuffer': opts.noresizebuffer,
        'http_chunk_size': opts.http_chunk_size,
        'continuedl': opts.continue_dl,
        'noprogress': opts.noprogress,
        'progress_with_newline': opts.progress_with_newline,
        'playliststart': opts.playliststart,
        'playlistend': opts.playlistend,
        'playlistreverse': opts.playlist_reverse,
        'playlistrandom': opts.playlist_random,
        'noplaylist': opts.noplaylist,
        'logtostderr': opts.outtmpl == '-',
        'consoletitle': opts.consoletitle,
        'nopart': opts.nopart,
        'updatetime': opts.updatetime,
        'writedescription': opts.writedescription,
        'writeannotations': opts.writeannotations,
        'writeinfojson': opts.writeinfojson,
        'writethumbnail': opts.writethumbnail,
        'write_all_thumbnails': opts.write_all_thumbnails,
        'writesubtitles': opts.writesubtitles,
        'writeautomaticsub': opts.writeautomaticsub,
        'allsubtitles': opts.allsubtitles,
        'listsubtitles': opts.listsubtitles,
        'subtitlesformat': opts.subtitlesformat,
        'subtitleslangs': opts.subtitleslangs,
        'matchtitle': decodeOption(opts.matchtitle),
        'rejecttitle': decodeOption(opts.rejecttitle),
        'max_downloads': opts.max_downloads,
        'prefer_free_formats': opts.prefer_free_formats,
        'verbose': opts.verbose,
        'dump_intermediate_pages': opts.dump_intermediate_pages,
        'write_pages': opts.write_pages,
        'test': opts.test,
        'keepvideo': opts.keepvideo,
        'min_filesize': opts.min_filesize,
        'max_filesize': opts.max_filesize,
        'min_views': opts.min_views,
        'max_views': opts.max_views,
        'daterange': date,
        'cachedir': opts.cachedir,
        'youtube_print_sig_code': opts.youtube_print_sig_code,
        'age_limit': opts.age_limit,
        'download_archive': download_archive_fn,
        'cookiefile': opts.cookiefile,
        'nocheckcertificate': opts.no_check_certificate,
        'prefer_insecure': opts.prefer_insecure,
        'proxy': opts.proxy,
        'socket_timeout': opts.socket_timeout,
        'bidi_workaround': opts.bidi_workaround,
        'debug_printtraffic': opts.debug_printtraffic,
        'prefer_ffmpeg': opts.prefer_ffmpeg,
        'include_ads': opts.include_ads,
        'default_search': opts.default_search,
        'youtube_include_dash_manifest': opts.youtube_include_dash_manifest,
        'encoding': opts.encoding,
        'extract_flat': opts.extract_flat,
        'mark_watched': opts.mark_watched,
        'merge_output_format': opts.merge_output_format,
        'postprocessors': postprocessors,
        'fixup': opts.fixup,
        'source_address': opts.source_address,
        'call_home': opts.call_home,
        'sleep_interval': opts.sleep_interval,
        'max_sleep_interval': opts.max_sleep_interval,
        'external_downloader': opts.external_downloader,
        'list_thumbnails': opts.list_thumbnails,
        'playlist_items': opts.playlist_items,
        'xattr_set_filesize': opts.xattr_set_filesize,
        'match_filter': match_filter,
        'no_color': opts.no_color,
        'ffmpeg_location': opts.ffmpeg_location,
        'hls_prefer_native': opts.hls_prefer_native,
        'hls_use_mpegts': opts.hls_use_mpegts,
        'external_downloader_args': external_downloader_args,
        'postprocessor_args': postprocessor_args,
        'cn_verification_proxy': opts.cn_verification_proxy,
        'geo_verification_proxy': opts.geo_verification_proxy,
        'config_location': opts.config_location,
        'geo_bypass': opts.geo_bypass,
        'geo_bypass_country': opts.geo_bypass_country,
        'geo_bypass_ip_block': opts.geo_bypass_ip_block,
        # just for deprecation check
        'autonumber': opts.autonumber if opts.autonumber is True else None,
        'usetitle': opts.usetitle if opts.usetitle is True else None,
    }

    with YoutubeDL(ydl_opts) as ydl:
        # Update version
        if opts.update_self:
            update_self(ydl.to_screen, opts.verbose, ydl._opener)

        # Remove cache dir
        if opts.rm_cachedir:
            ydl.cache.remove()

        # Maybe do nothing
        if (len(all_urls) < 1) and (opts.load_info_filename is None):
            if opts.update_self or opts.rm_cachedir:
                sys.exit()

            ydl.warn_if_short_id(sys.argv[1:] if argv is None else argv)
            parser.error(
                'You must provide at least one URL.\n'
                'Type youtube-dl --help to see a list of all options.')

        try:
            if opts.load_info_filename is not None:
                retcode = ydl.download_with_info_file(expand_path(opts.load_info_filename))
            else:
                retcode = ydl.download(all_urls)
        except MaxDownloadsReached:
            ydl.to_screen('--max-download limit reached, aborting.')
            retcode = 101

    sys.exit(retcode)


def main(argv=None):
    try:
        _real_main(argv)
    except DownloadError:
        sys.exit(1)
    except SameFileError:
        sys.exit('ERROR: fixed output name but more than one file to download')
    except KeyboardInterrupt:
        sys.exit('\nERROR: Interrupted by user')


__all__ = ['main', 'YoutubeDL', 'gen_extractors', 'list_extractors']
from __future__ import unicode_literals

import json
import operator
import re

from .utils import (
    ExtractorError,
    remove_quotes,
)

_OPERATORS = [
    ('|', operator.or_),
    ('^', operator.xor),
    ('&', operator.and_),
    ('>>', operator.rshift),
    ('<<', operator.lshift),
    ('-', operator.sub),
    ('+', operator.add),
    ('%', operator.mod),
    ('/', operator.truediv),
    ('*', operator.mul),
]
_ASSIGN_OPERATORS = [(op + '=', opfunc) for op, opfunc in _OPERATORS]
_ASSIGN_OPERATORS.append(('=', lambda cur, right: right))

_NAME_RE = r'[a-zA-Z_$][a-zA-Z_$0-9]*'


class JSInterpreter(object):
    def __init__(self, code, objects=None):
        if objects is None:
            objects = {}
        self.code = code
        self._functions = {}
        self._objects = objects

    def interpret_statement(self, stmt, local_vars, allow_recursion=100):
        if allow_recursion < 0:
            raise ExtractorError('Recursion limit reached')

        should_abort = False
        stmt = stmt.lstrip()
        stmt_m = re.match(r'var\s', stmt)
        if stmt_m:
            expr = stmt[len(stmt_m.group(0)):]
        else:
            return_m = re.match(r'return(?:\s+|$)', stmt)
            if return_m:
                expr = stmt[len(return_m.group(0)):]
                should_abort = True
            else:
                # Try interpreting it as an expression
                expr = stmt

        v = self.interpret_expression(expr, local_vars, allow_recursion)
        return v, should_abort

    def interpret_expression(self, expr, local_vars, allow_recursion):
        expr = expr.strip()
        if expr == '':  # Empty expression
            return None

        if expr.startswith('('):
            parens_count = 0
            for m in re.finditer(r'[()]', expr):
                if m.group(0) == '(':
                    parens_count += 1
                else:
                    parens_count -= 1
                    if parens_count == 0:
                        sub_expr = expr[1:m.start()]
                        sub_result = self.interpret_expression(
                            sub_expr, local_vars, allow_recursion)
                        remaining_expr = expr[m.end():].strip()
                        if not remaining_expr:
                            return sub_result
                        else:
                            expr = json.dumps(sub_result) + remaining_expr
                        break
            else:
                raise ExtractorError('Premature end of parens in %r' % expr)

        for op, opfunc in _ASSIGN_OPERATORS:
            m = re.match(r'''(?x)
                (?P<out>%s)(?:\[(?P<index>[^\]]+?)\])?
                \s*%s
                (?P<expr>.*)$''' % (_NAME_RE, re.escape(op)), expr)
            if not m:
                continue
            right_val = self.interpret_expression(
                m.group('expr'), local_vars, allow_recursion - 1)

            if m.groupdict().get('index'):
                lvar = local_vars[m.group('out')]
                idx = self.interpret_expression(
                    m.group('index'), local_vars, allow_recursion)
                assert isinstance(idx, int)
                cur = lvar[idx]
                val = opfunc(cur, right_val)
                lvar[idx] = val
                return val
            else:
                cur = local_vars.get(m.group('out'))
                val = opfunc(cur, right_val)
                local_vars[m.group('out')] = val
                return val

        if expr.isdigit():
            return int(expr)

        var_m = re.match(
            r'(?!if|return|true|false)(?P<name>%s)$' % _NAME_RE,
            expr)
        if var_m:
            return local_vars[var_m.group('name')]

        try:
            return json.loads(expr)
        except ValueError:
            pass

        m = re.match(
            r'(?P<in>%s)\[(?P<idx>.+)\]$' % _NAME_RE, expr)
        if m:
            val = local_vars[m.group('in')]
            idx = self.interpret_expression(
                m.group('idx'), local_vars, allow_recursion - 1)
            return val[idx]

        m = re.match(
            r'(?P<var>%s)(?:\.(?P<member>[^(]+)|\[(?P<member2>[^]]+)\])\s*(?:\(+(?P<args>[^()]*)\))?$' % _NAME_RE,
            expr)
        if m:
            variable = m.group('var')
            member = remove_quotes(m.group('member') or m.group('member2'))
            arg_str = m.group('args')

            if variable in local_vars:
                obj = local_vars[variable]
            else:
                if variable not in self._objects:
                    self._objects[variable] = self.extract_object(variable)
                obj = self._objects[variable]

            if arg_str is None:
                # Member access
                if member == 'length':
                    return len(obj)
                return obj[member]

            assert expr.endswith(')')
            # Function call
            if arg_str == '':
                argvals = tuple()
            else:
                argvals = tuple([
                    self.interpret_expression(v, local_vars, allow_recursion)
                    for v in arg_str.split(',')])

            if member == 'split':
                assert argvals == ('',)
                return list(obj)
            if member == 'join':
                assert len(argvals) == 1
                return argvals[0].join(obj)
            if member == 'reverse':
                assert len(argvals) == 0
                obj.reverse()
                return obj
            if member == 'slice':
                assert len(argvals) == 1
                return obj[argvals[0]:]
            if member == 'splice':
                assert isinstance(obj, list)
                index, howMany = argvals
                res = []
                for i in range(index, min(index + howMany, len(obj))):
                    res.append(obj.pop(index))
                return res

            return obj[member](argvals)

        for op, opfunc in _OPERATORS:
            m = re.match(r'(?P<x>.+?)%s(?P<y>.+)' % re.escape(op), expr)
            if not m:
                continue
            x, abort = self.interpret_statement(
                m.group('x'), local_vars, allow_recursion - 1)
            if abort:
                raise ExtractorError(
                    'Premature left-side return of %s in %r' % (op, expr))
            y, abort = self.interpret_statement(
                m.group('y'), local_vars, allow_recursion - 1)
            if abort:
                raise ExtractorError(
                    'Premature right-side return of %s in %r' % (op, expr))
            return opfunc(x, y)

        m = re.match(
            r'^(?P<func>%s)\((?P<args>[a-zA-Z0-9_$,]*)\)$' % _NAME_RE, expr)
        if m:
            fname = m.group('func')
            argvals = tuple([
                int(v) if v.isdigit() else local_vars[v]
                for v in m.group('args').split(',')]) if len(m.group('args')) > 0 else tuple()
            if fname not in self._functions:
                self._functions[fname] = self.extract_function(fname)
            return self._functions[fname](argvals)

        raise ExtractorError('Unsupported JS expression %r' % expr)

    def extract_object(self, objname):
        _FUNC_NAME_RE = r'''(?:[a-zA-Z$0-9]+|"[a-zA-Z$0-9]+"|'[a-zA-Z$0-9]+')'''
        obj = {}
        obj_m = re.search(
            r'''(?x)
                (?<!this\.)%s\s*=\s*{\s*
                    (?P<fields>(%s\s*:\s*function\s*\(.*?\)\s*{.*?}(?:,\s*)?)*)
                }\s*;
            ''' % (re.escape(objname), _FUNC_NAME_RE),
            self.code)
        fields = obj_m.group('fields')
        # Currently, it only supports function definitions
        fields_m = re.finditer(
            r'''(?x)
                (?P<key>%s)\s*:\s*function\s*\((?P<args>[a-z,]+)\){(?P<code>[^}]+)}
            ''' % _FUNC_NAME_RE,
            fields)
        for f in fields_m:
            argnames = f.group('args').split(',')
            obj[remove_quotes(f.group('key'))] = self.build_function(argnames, f.group('code'))

        return obj

    def extract_function(self, funcname):
        func_m = re.search(
            r'''(?x)
                (?:function\s+%s|[{;,]\s*%s\s*=\s*function|var\s+%s\s*=\s*function)\s*
                \((?P<args>[^)]*)\)\s*
                \{(?P<code>[^}]+)\}''' % (
                re.escape(funcname), re.escape(funcname), re.escape(funcname)),
            self.code)
        if func_m is None:
            raise ExtractorError('Could not find JS function %r' % funcname)
        argnames = func_m.group('args').split(',')

        return self.build_function(argnames, func_m.group('code'))

    def call_function(self, funcname, *args):
        f = self.extract_function(funcname)
        return f(args)

    def build_function(self, argnames, code):
        def resf(args):
            local_vars = dict(zip(argnames, args))
            for stmt in code.split(';'):
                res, abort = self.interpret_statement(stmt, local_vars)
                if abort:
                    break
            return res
        return resf
#!/usr/bin/env python
from __future__ import unicode_literals

# Execute with
# $ python youtube_dl/__main__.py (2.6+)
# $ python -m youtube_dl          (2.7+)

import sys

if __package__ is None and not hasattr(sys, 'frozen'):
    # direct call of __main__.py
    import os.path
    path = os.path.realpath(os.path.abspath(__file__))
    sys.path.insert(0, os.path.dirname(os.path.dirname(path)))

import youtube_dl

if __name__ == '__main__':
    youtube_dl.main()
from __future__ import unicode_literals

import os.path
import optparse
import re
import sys

from .downloader.external import list_external_downloaders
from .compat import (
    compat_expanduser,
    compat_get_terminal_size,
    compat_getenv,
    compat_kwargs,
    compat_shlex_split,
)
from .utils import (
    preferredencoding,
    write_string,
)
from .version import __version__


def _hide_login_info(opts):
    PRIVATE_OPTS = set(['-p', '--password', '-u', '--username', '--video-password', '--ap-password', '--ap-username'])
    eqre = re.compile('^(?P<key>' + ('|'.join(re.escape(po) for po in PRIVATE_OPTS)) + ')=.+$')

    def _scrub_eq(o):
        m = eqre.match(o)
        if m:
            return m.group('key') + '=PRIVATE'
        else:
            return o

    opts = list(map(_scrub_eq, opts))
    for idx, opt in enumerate(opts):
        if opt in PRIVATE_OPTS and idx + 1 < len(opts):
            opts[idx + 1] = 'PRIVATE'
    return opts


def parseOpts(overrideArguments=None):
    def _readOptions(filename_bytes, default=[]):
        try:
            optionf = open(filename_bytes)
        except IOError:
            return default  # silently skip if file is not present
        try:
            # FIXME: https://github.com/ytdl-org/youtube-dl/commit/dfe5fa49aed02cf36ba9f743b11b0903554b5e56
            contents = optionf.read()
            if sys.version_info < (3,):
                contents = contents.decode(preferredencoding())
            res = compat_shlex_split(contents, comments=True)
        finally:
            optionf.close()
        return res

    def _readUserConf():
        xdg_config_home = compat_getenv('XDG_CONFIG_HOME')
        if xdg_config_home:
            userConfFile = os.path.join(xdg_config_home, 'youtube-dl', 'config')
            if not os.path.isfile(userConfFile):
                userConfFile = os.path.join(xdg_config_home, 'youtube-dl.conf')
        else:
            userConfFile = os.path.join(compat_expanduser('~'), '.config', 'youtube-dl', 'config')
            if not os.path.isfile(userConfFile):
                userConfFile = os.path.join(compat_expanduser('~'), '.config', 'youtube-dl.conf')
        userConf = _readOptions(userConfFile, None)

        if userConf is None:
            appdata_dir = compat_getenv('appdata')
            if appdata_dir:
                userConf = _readOptions(
                    os.path.join(appdata_dir, 'youtube-dl', 'config'),
                    default=None)
                if userConf is None:
                    userConf = _readOptions(
                        os.path.join(appdata_dir, 'youtube-dl', 'config.txt'),
                        default=None)

        if userConf is None:
            userConf = _readOptions(
                os.path.join(compat_expanduser('~'), 'youtube-dl.conf'),
                default=None)
        if userConf is None:
            userConf = _readOptions(
                os.path.join(compat_expanduser('~'), 'youtube-dl.conf.txt'),
                default=None)

        if userConf is None:
            userConf = []

        return userConf

    def _format_option_string(option):
        ''' ('-o', '--option') -> -o, --format METAVAR'''

        opts = []

        if option._short_opts:
            opts.append(option._short_opts[0])
        if option._long_opts:
            opts.append(option._long_opts[0])
        if len(opts) > 1:
            opts.insert(1, ', ')

        if option.takes_value():
            opts.append(' %s' % option.metavar)

        return ''.join(opts)

    def _comma_separated_values_options_callback(option, opt_str, value, parser):
        setattr(parser.values, option.dest, value.split(','))

    # No need to wrap help messages if we're on a wide console
    columns = compat_get_terminal_size().columns
    max_width = columns if columns else 80
    max_help_position = 80

    fmt = optparse.IndentedHelpFormatter(width=max_width, max_help_position=max_help_position)
    fmt.format_option_strings = _format_option_string

    kw = {
        'version': __version__,
        'formatter': fmt,
        'usage': '%prog [OPTIONS] URL [URL...]',
        'conflict_handler': 'resolve',
    }

    parser = optparse.OptionParser(**compat_kwargs(kw))

    general = optparse.OptionGroup(parser, 'General Options')
    general.add_option(
        '-h', '--help',
        action='help',
        help='Print this help text and exit')
    general.add_option(
        '--version',
        action='version',
        help='Print program version and exit')
    general.add_option(
        '-U', '--update',
        action='store_true', dest='update_self',
        help='Update this program to latest version. Make sure that you have sufficient permissions (run with sudo if needed)')
    general.add_option(
        '-i', '--ignore-errors',
        action='store_true', dest='ignoreerrors', default=False,
        help='Continue on download errors, for example to skip unavailable videos in a playlist')
    general.add_option(
        '--abort-on-error',
        action='store_false', dest='ignoreerrors',
        help='Abort downloading of further videos (in the playlist or the command line) if an error occurs')
    general.add_option(
        '--dump-user-agent',
        action='store_true', dest='dump_user_agent', default=False,
        help='Display the current browser identification')
    general.add_option(
        '--list-extractors',
        action='store_true', dest='list_extractors', default=False,
        help='List all supported extractors')
    general.add_option(
        '--extractor-descriptions',
        action='store_true', dest='list_extractor_descriptions', default=False,
        help='Output descriptions of all supported extractors')
    general.add_option(
        '--force-generic-extractor',
        action='store_true', dest='force_generic_extractor', default=False,
        help='Force extraction to use the generic extractor')
    general.add_option(
        '--default-search',
        dest='default_search', metavar='PREFIX',
        help='Use this prefix for unqualified URLs. For example "gvsearch2:" downloads two videos from google videos for youtube-dl "large apple". Use the value "auto" to let youtube-dl guess ("auto_warning" to emit a warning when guessing). "error" just throws an error. The default value "fixup_error" repairs broken URLs, but emits an error if this is not possible instead of searching.')
    general.add_option(
        '--ignore-config',
        action='store_true',
        help='Do not read configuration files. '
        'When given in the global configuration file /etc/youtube-dl.conf: '
        'Do not read the user configuration in ~/.config/youtube-dl/config '
        '(%APPDATA%/youtube-dl/config.txt on Windows)')
    general.add_option(
        '--config-location',
        dest='config_location', metavar='PATH',
        help='Location of the configuration file; either the path to the config or its containing directory.')
    general.add_option(
        '--flat-playlist',
        action='store_const', dest='extract_flat', const='in_playlist',
        default=False,
        help='Do not extract the videos of a playlist, only list them.')
    general.add_option(
        '--mark-watched',
        action='store_true', dest='mark_watched', default=False,
        help='Mark videos watched (YouTube only)')
    general.add_option(
        '--no-mark-watched',
        action='store_false', dest='mark_watched', default=False,
        help='Do not mark videos watched (YouTube only)')
    general.add_option(
        '--no-color', '--no-colors',
        action='store_true', dest='no_color',
        default=False,
        help='Do not emit color codes in output')

    network = optparse.OptionGroup(parser, 'Network Options')
    network.add_option(
        '--proxy', dest='proxy',
        default=None, metavar='URL',
        help='Use the specified HTTP/HTTPS/SOCKS proxy. To enable '
             'SOCKS proxy, specify a proper scheme. For example '
             'socks5://127.0.0.1:1080/. Pass in an empty string (--proxy "") '
             'for direct connection')
    network.add_option(
        '--socket-timeout',
        dest='socket_timeout', type=float, default=None, metavar='SECONDS',
        help='Time to wait before giving up, in seconds')
    network.add_option(
        '--source-address',
        metavar='IP', dest='source_address', default=None,
        help='Client-side IP address to bind to',
    )
    network.add_option(
        '-4', '--force-ipv4',
        action='store_const', const='0.0.0.0', dest='source_address',
        help='Make all connections via IPv4',
    )
    network.add_option(
        '-6', '--force-ipv6',
        action='store_const', const='::', dest='source_address',
        help='Make all connections via IPv6',
    )

    geo = optparse.OptionGroup(parser, 'Geo Restriction')
    geo.add_option(
        '--geo-verification-proxy',
        dest='geo_verification_proxy', default=None, metavar='URL',
        help='Use this proxy to verify the IP address for some geo-restricted sites. '
        'The default proxy specified by --proxy (or none, if the option is not present) is used for the actual downloading.')
    geo.add_option(
        '--cn-verification-proxy',
        dest='cn_verification_proxy', default=None, metavar='URL',
        help=optparse.SUPPRESS_HELP)
    geo.add_option(
        '--geo-bypass',
        action='store_true', dest='geo_bypass', default=True,
        help='Bypass geographic restriction via faking X-Forwarded-For HTTP header')
    geo.add_option(
        '--no-geo-bypass',
        action='store_false', dest='geo_bypass', default=True,
        help='Do not bypass geographic restriction via faking X-Forwarded-For HTTP header')
    geo.add_option(
        '--geo-bypass-country', metavar='CODE',
        dest='geo_bypass_country', default=None,
        help='Force bypass geographic restriction with explicitly provided two-letter ISO 3166-2 country code')
    geo.add_option(
        '--geo-bypass-ip-block', metavar='IP_BLOCK',
        dest='geo_bypass_ip_block', default=None,
        help='Force bypass geographic restriction with explicitly provided IP block in CIDR notation')

    selection = optparse.OptionGroup(parser, 'Video Selection')
    selection.add_option(
        '--playlist-start',
        dest='playliststart', metavar='NUMBER', default=1, type=int,
        help='Playlist video to start at (default is %default)')
    selection.add_option(
        '--playlist-end',
        dest='playlistend', metavar='NUMBER', default=None, type=int,
        help='Playlist video to end at (default is last)')
    selection.add_option(
        '--playlist-items',
        dest='playlist_items', metavar='ITEM_SPEC', default=None,
        help='Playlist video items to download. Specify indices of the videos in the playlist separated by commas like: "--playlist-items 1,2,5,8" if you want to download videos indexed 1, 2, 5, 8 in the playlist. You can specify range: "--playlist-items 1-3,7,10-13", it will download the videos at index 1, 2, 3, 7, 10, 11, 12 and 13.')
    selection.add_option(
        '--match-title',
        dest='matchtitle', metavar='REGEX',
        help='Download only matching titles (regex or caseless sub-string)')
    selection.add_option(
        '--reject-title',
        dest='rejecttitle', metavar='REGEX',
        help='Skip download for matching titles (regex or caseless sub-string)')
    selection.add_option(
        '--max-downloads',
        dest='max_downloads', metavar='NUMBER', type=int, default=None,
        help='Abort after downloading NUMBER files')
    selection.add_option(
        '--min-filesize',
        metavar='SIZE', dest='min_filesize', default=None,
        help='Do not download any videos smaller than SIZE (e.g. 50k or 44.6m)')
    selection.add_option(
        '--max-filesize',
        metavar='SIZE', dest='max_filesize', default=None,
        help='Do not download any videos larger than SIZE (e.g. 50k or 44.6m)')
    selection.add_option(
        '--date',
        metavar='DATE', dest='date', default=None,
        help='Download only videos uploaded in this date')
    selection.add_option(
        '--datebefore',
        metavar='DATE', dest='datebefore', default=None,
        help='Download only videos uploaded on or before this date (i.e. inclusive)')
    selection.add_option(
        '--dateafter',
        metavar='DATE', dest='dateafter', default=None,
        help='Download only videos uploaded on or after this date (i.e. inclusive)')
    selection.add_option(
        '--min-views',
        metavar='COUNT', dest='min_views', default=None, type=int,
        help='Do not download any videos with less than COUNT views')
    selection.add_option(
        '--max-views',
        metavar='COUNT', dest='max_views', default=None, type=int,
        help='Do not download any videos with more than COUNT views')
    selection.add_option(
        '--match-filter',
        metavar='FILTER', dest='match_filter', default=None,
        help=(
            'Generic video filter. '
            'Specify any key (see the "OUTPUT TEMPLATE" for a list of available keys) to '
            'match if the key is present, '
            '!key to check if the key is not present, '
            'key > NUMBER (like "comment_count > 12", also works with '
            '>=, <, <=, !=, =) to compare against a number, '
            'key = \'LITERAL\' (like "uploader = \'Mike Smith\'", also works with !=) '
            'to match against a string literal '
            'and & to require multiple matches. '
            'Values which are not known are excluded unless you '
            'put a question mark (?) after the operator. '
            'For example, to only match videos that have been liked more than '
            '100 times and disliked less than 50 times (or the dislike '
            'functionality is not available at the given service), but who '
            'also have a description, use --match-filter '
            '"like_count > 100 & dislike_count <? 50 & description" .'
        ))
    selection.add_option(
        '--no-playlist',
        action='store_true', dest='noplaylist', default=False,
        help='Download only the video, if the URL refers to a video and a playlist.')
    selection.add_option(
        '--yes-playlist',
        action='store_false', dest='noplaylist', default=False,
        help='Download the playlist, if the URL refers to a video and a playlist.')
    selection.add_option(
        '--age-limit',
        metavar='YEARS', dest='age_limit', default=None, type=int,
        help='Download only videos suitable for the given age')
    selection.add_option(
        '--download-archive', metavar='FILE',
        dest='download_archive',
        help='Download only videos not listed in the archive file. Record the IDs of all downloaded videos in it.')
    selection.add_option(
        '--include-ads',
        dest='include_ads', action='store_true',
        help='Download advertisements as well (experimental)')

    authentication = optparse.OptionGroup(parser, 'Authentication Options')
    authentication.add_option(
        '-u', '--username',
        dest='username', metavar='USERNAME',
        help='Login with this account ID')
    authentication.add_option(
        '-p', '--password',
        dest='password', metavar='PASSWORD',
        help='Account password. If this option is left out, youtube-dl will ask interactively.')
    authentication.add_option(
        '-2', '--twofactor',
        dest='twofactor', metavar='TWOFACTOR',
        help='Two-factor authentication code')
    authentication.add_option(
        '-n', '--netrc',
        action='store_true', dest='usenetrc', default=False,
        help='Use .netrc authentication data')
    authentication.add_option(
        '--video-password',
        dest='videopassword', metavar='PASSWORD',
        help='Video password (vimeo, smotri, youku)')

    adobe_pass = optparse.OptionGroup(parser, 'Adobe Pass Options')
    adobe_pass.add_option(
        '--ap-mso',
        dest='ap_mso', metavar='MSO',
        help='Adobe Pass multiple-system operator (TV provider) identifier, use --ap-list-mso for a list of available MSOs')
    adobe_pass.add_option(
        '--ap-username',
        dest='ap_username', metavar='USERNAME',
        help='Multiple-system operator account login')
    adobe_pass.add_option(
        '--ap-password',
        dest='ap_password', metavar='PASSWORD',
        help='Multiple-system operator account password. If this option is left out, youtube-dl will ask interactively.')
    adobe_pass.add_option(
        '--ap-list-mso',
        action='store_true', dest='ap_list_mso', default=False,
        help='List all supported multiple-system operators')

    video_format = optparse.OptionGroup(parser, 'Video Format Options')
    video_format.add_option(
        '-f', '--format',
        action='store', dest='format', metavar='FORMAT', default=None,
        help='Video format code, see the "FORMAT SELECTION" for all the info')
    video_format.add_option(
        '--all-formats',
        action='store_const', dest='format', const='all',
        help='Download all available video formats')
    video_format.add_option(
        '--prefer-free-formats',
        action='store_true', dest='prefer_free_formats', default=False,
        help='Prefer free video formats unless a specific one is requested')
    video_format.add_option(
        '-F', '--list-formats',
        action='store_true', dest='listformats',
        help='List all available formats of requested videos')
    video_format.add_option(
        '--youtube-include-dash-manifest',
        action='store_true', dest='youtube_include_dash_manifest', default=True,
        help=optparse.SUPPRESS_HELP)
    video_format.add_option(
        '--youtube-skip-dash-manifest',
        action='store_false', dest='youtube_include_dash_manifest',
        help='Do not download the DASH manifests and related data on YouTube videos')
    video_format.add_option(
        '--merge-output-format',
        action='store', dest='merge_output_format', metavar='FORMAT', default=None,
        help=(
            'If a merge is required (e.g. bestvideo+bestaudio), '
            'output to given container format. One of mkv, mp4, ogg, webm, flv. '
            'Ignored if no merge is required'))

    subtitles = optparse.OptionGroup(parser, 'Subtitle Options')
    subtitles.add_option(
        '--write-sub', '--write-srt',
        action='store_true', dest='writesubtitles', default=False,
        help='Write subtitle file')
    subtitles.add_option(
        '--write-auto-sub', '--write-automatic-sub',
        action='store_true', dest='writeautomaticsub', default=False,
        help='Write automatically generated subtitle file (YouTube only)')
    subtitles.add_option(
        '--all-subs',
        action='store_true', dest='allsubtitles', default=False,
        help='Download all the available subtitles of the video')
    subtitles.add_option(
        '--list-subs',
        action='store_true', dest='listsubtitles', default=False,
        help='List all available subtitles for the video')
    subtitles.add_option(
        '--sub-format',
        action='store', dest='subtitlesformat', metavar='FORMAT', default='best',
        help='Subtitle format, accepts formats preference, for example: "srt" or "ass/srt/best"')
    subtitles.add_option(
        '--sub-lang', '--sub-langs', '--srt-lang',
        action='callback', dest='subtitleslangs', metavar='LANGS', type='str',
        default=[], callback=_comma_separated_values_options_callback,
        help='Languages of the subtitles to download (optional) separated by commas, use --list-subs for available language tags')

    downloader = optparse.OptionGroup(parser, 'Download Options')
    downloader.add_option(
        '-r', '--limit-rate', '--rate-limit',
        dest='ratelimit', metavar='RATE',
        help='Maximum download rate in bytes per second (e.g. 50K or 4.2M)')
    downloader.add_option(
        '-R', '--retries',
        dest='retries', metavar='RETRIES', default=10,
        help='Number of retries (default is %default), or "infinite".')
    downloader.add_option(
        '--fragment-retries',
        dest='fragment_retries', metavar='RETRIES', default=10,
        help='Number of retries for a fragment (default is %default), or "infinite" (DASH, hlsnative and ISM)')
    downloader.add_option(
        '--skip-unavailable-fragments',
        action='store_true', dest='skip_unavailable_fragments', default=True,
        help='Skip unavailable fragments (DASH, hlsnative and ISM)')
    downloader.add_option(
        '--abort-on-unavailable-fragment',
        action='store_false', dest='skip_unavailable_fragments',
        help='Abort downloading when some fragment is not available')
    downloader.add_option(
        '--keep-fragments',
        action='store_true', dest='keep_fragments', default=False,
        help='Keep downloaded fragments on disk after downloading is finished; fragments are erased by default')
    downloader.add_option(
        '--buffer-size',
        dest='buffersize', metavar='SIZE', default='1024',
        help='Size of download buffer (e.g. 1024 or 16K) (default is %default)')
    downloader.add_option(
        '--no-resize-buffer',
        action='store_true', dest='noresizebuffer', default=False,
        help='Do not automatically adjust the buffer size. By default, the buffer size is automatically resized from an initial value of SIZE.')
    downloader.add_option(
        '--http-chunk-size',
        dest='http_chunk_size', metavar='SIZE', default=None,
        help='Size of a chunk for chunk-based HTTP downloading (e.g. 10485760 or 10M) (default is disabled). '
             'May be useful for bypassing bandwidth throttling imposed by a webserver (experimental)')
    downloader.add_option(
        '--test',
        action='store_true', dest='test', default=False,
        help=optparse.SUPPRESS_HELP)
    downloader.add_option(
        '--playlist-reverse',
        action='store_true',
        help='Download playlist videos in reverse order')
    downloader.add_option(
        '--playlist-random',
        action='store_true',
        help='Download playlist videos in random order')
    downloader.add_option(
        '--xattr-set-filesize',
        dest='xattr_set_filesize', action='store_true',
        help='Set file xattribute ytdl.filesize with expected file size')
    downloader.add_option(
        '--hls-prefer-native',
        dest='hls_prefer_native', action='store_true', default=None,
        help='Use the native HLS downloader instead of ffmpeg')
    downloader.add_option(
        '--hls-prefer-ffmpeg',
        dest='hls_prefer_native', action='store_false', default=None,
        help='Use ffmpeg instead of the native HLS downloader')
    downloader.add_option(
        '--hls-use-mpegts',
        dest='hls_use_mpegts', action='store_true',
        help='Use the mpegts container for HLS videos, allowing to play the '
             'video while downloading (some players may not be able to play it)')
    downloader.add_option(
        '--external-downloader',
        dest='external_downloader', metavar='COMMAND',
        help='Use the specified external downloader. '
             'Currently supports %s' % ','.join(list_external_downloaders()))
    downloader.add_option(
        '--external-downloader-args',
        dest='external_downloader_args', metavar='ARGS',
        help='Give these arguments to the external downloader')

    workarounds = optparse.OptionGroup(parser, 'Workarounds')
    workarounds.add_option(
        '--encoding',
        dest='encoding', metavar='ENCODING',
        help='Force the specified encoding (experimental)')
    workarounds.add_option(
        '--no-check-certificate',
        action='store_true', dest='no_check_certificate', default=False,
        help='Suppress HTTPS certificate validation')
    workarounds.add_option(
        '--prefer-insecure',
        '--prefer-unsecure', action='store_true', dest='prefer_insecure',
        help='Use an unencrypted connection to retrieve information about the video. (Currently supported only for YouTube)')
    workarounds.add_option(
        '--user-agent',
        metavar='UA', dest='user_agent',
        help='Specify a custom user agent')
    workarounds.add_option(
        '--referer',
        metavar='URL', dest='referer', default=None,
        help='Specify a custom referer, use if the video access is restricted to one domain',
    )
    workarounds.add_option(
        '--add-header',
        metavar='FIELD:VALUE', dest='headers', action='append',
        help='Specify a custom HTTP header and its value, separated by a colon \':\'. You can use this option multiple times',
    )
    workarounds.add_option(
        '--bidi-workaround',
        dest='bidi_workaround', action='store_true',
        help='Work around terminals that lack bidirectional text support. Requires bidiv or fribidi executable in PATH')
    workarounds.add_option(
        '--sleep-interval', '--min-sleep-interval', metavar='SECONDS',
        dest='sleep_interval', type=float,
        help=(
            'Number of seconds to sleep before each download when used alone '
            'or a lower bound of a range for randomized sleep before each download '
            '(minimum possible number of seconds to sleep) when used along with '
            '--max-sleep-interval.'))
    workarounds.add_option(
        '--max-sleep-interval', metavar='SECONDS',
        dest='max_sleep_interval', type=float,
        help=(
            'Upper bound of a range for randomized sleep before each download '
            '(maximum possible number of seconds to sleep). Must only be used '
            'along with --min-sleep-interval.'))

    verbosity = optparse.OptionGroup(parser, 'Verbosity / Simulation Options')
    verbosity.add_option(
        '-q', '--quiet',
        action='store_true', dest='quiet', default=False,
        help='Activate quiet mode')
    verbosity.add_option(
        '--no-warnings',
        dest='no_warnings', action='store_true', default=False,
        help='Ignore warnings')
    verbosity.add_option(
        '-s', '--simulate',
        action='store_true', dest='simulate', default=False,
        help='Do not download the video and do not write anything to disk')
    verbosity.add_option(
        '--skip-download',
        action='store_true', dest='skip_download', default=False,
        help='Do not download the video')
    verbosity.add_option(
        '-g', '--get-url',
        action='store_true', dest='geturl', default=False,
        help='Simulate, quiet but print URL')
    verbosity.add_option(
        '-e', '--get-title',
        action='store_true', dest='gettitle', default=False,
        help='Simulate, quiet but print title')
    verbosity.add_option(
        '--get-id',
        action='store_true', dest='getid', default=False,
        help='Simulate, quiet but print id')
    verbosity.add_option(
        '--get-thumbnail',
        action='store_true', dest='getthumbnail', default=False,
        help='Simulate, quiet but print thumbnail URL')
    verbosity.add_option(
        '--get-description',
        action='store_true', dest='getdescription', default=False,
        help='Simulate, quiet but print video description')
    verbosity.add_option(
        '--get-duration',
        action='store_true', dest='getduration', default=False,
        help='Simulate, quiet but print video length')
    verbosity.add_option(
        '--get-filename',
        action='store_true', dest='getfilename', default=False,
        help='Simulate, quiet but print output filename')
    verbosity.add_option(
        '--get-format',
        action='store_true', dest='getformat', default=False,
        help='Simulate, quiet but print output format')
    verbosity.add_option(
        '-j', '--dump-json',
        action='store_true', dest='dumpjson', default=False,
        help='Simulate, quiet but print JSON information. See the "OUTPUT TEMPLATE" for a description of available keys.')
    verbosity.add_option(
        '-J', '--dump-single-json',
        action='store_true', dest='dump_single_json', default=False,
        help='Simulate, quiet but print JSON information for each command-line argument. If the URL refers to a playlist, dump the whole playlist information in a single line.')
    verbosity.add_option(
        '--print-json',
        action='store_true', dest='print_json', default=False,
        help='Be quiet and print the video information as JSON (video is still being downloaded).',
    )
    verbosity.add_option(
        '--newline',
        action='store_true', dest='progress_with_newline', default=False,
        help='Output progress bar as new lines')
    verbosity.add_option(
        '--no-progress',
        action='store_true', dest='noprogress', default=False,
        help='Do not print progress bar')
    verbosity.add_option(
        '--console-title',
        action='store_true', dest='consoletitle', default=False,
        help='Display progress in console titlebar')
    verbosity.add_option(
        '-v', '--verbose',
        action='store_true', dest='verbose', default=False,
        help='Print various debugging information')
    verbosity.add_option(
        '--dump-pages', '--dump-intermediate-pages',
        action='store_true', dest='dump_intermediate_pages', default=False,
        help='Print downloaded pages encoded using base64 to debug problems (very verbose)')
    verbosity.add_option(
        '--write-pages',
        action='store_true', dest='write_pages', default=False,
        help='Write downloaded intermediary pages to files in the current directory to debug problems')
    verbosity.add_option(
        '--youtube-print-sig-code',
        action='store_true', dest='youtube_print_sig_code', default=False,
        help=optparse.SUPPRESS_HELP)
    verbosity.add_option(
        '--print-traffic', '--dump-headers',
        dest='debug_printtraffic', action='store_true', default=False,
        help='Display sent and read HTTP traffic')
    verbosity.add_option(
        '-C', '--call-home',
        dest='call_home', action='store_true', default=False,
        help='Contact the youtube-dl server for debugging')
    verbosity.add_option(
        '--no-call-home',
        dest='call_home', action='store_false', default=False,
        help='Do NOT contact the youtube-dl server for debugging')

    filesystem = optparse.OptionGroup(parser, 'Filesystem Options')
    filesystem.add_option(
        '-a', '--batch-file',
        dest='batchfile', metavar='FILE',
        help="File containing URLs to download ('-' for stdin), one URL per line. "
             "Lines starting with '#', ';' or ']' are considered as comments and ignored.")
    filesystem.add_option(
        '--id', default=False,
        action='store_true', dest='useid', help='Use only video ID in file name')
    filesystem.add_option(
        '-o', '--output',
        dest='outtmpl', metavar='TEMPLATE',
        help=('Output filename template, see the "OUTPUT TEMPLATE" for all the info'))
    filesystem.add_option(
        '--autonumber-size',
        dest='autonumber_size', metavar='NUMBER', type=int,
        help=optparse.SUPPRESS_HELP)
    filesystem.add_option(
        '--autonumber-start',
        dest='autonumber_start', metavar='NUMBER', default=1, type=int,
        help='Specify the start value for %(autonumber)s (default is %default)')
    filesystem.add_option(
        '--restrict-filenames',
        action='store_true', dest='restrictfilenames', default=False,
        help='Restrict filenames to only ASCII characters, and avoid "&" and spaces in filenames')
    filesystem.add_option(
        '-A', '--auto-number',
        action='store_true', dest='autonumber', default=False,
        help=optparse.SUPPRESS_HELP)
    filesystem.add_option(
        '-t', '--title',
        action='store_true', dest='usetitle', default=False,
        help=optparse.SUPPRESS_HELP)
    filesystem.add_option(
        '-l', '--literal', default=False,
        action='store_true', dest='usetitle',
        help=optparse.SUPPRESS_HELP)
    filesystem.add_option(
        '-w', '--no-overwrites',
        action='store_true', dest='nooverwrites', default=False,
        help='Do not overwrite files')
    filesystem.add_option(
        '-c', '--continue',
        action='store_true', dest='continue_dl', default=True,
        help='Force resume of partially downloaded files. By default, youtube-dl will resume downloads if possible.')
    filesystem.add_option(
        '--no-continue',
        action='store_false', dest='continue_dl',
        help='Do not resume partially downloaded files (restart from beginning)')
    filesystem.add_option(
        '--no-part',
        action='store_true', dest='nopart', default=False,
        help='Do not use .part files - write directly into output file')
    filesystem.add_option(
        '--no-mtime',
        action='store_false', dest='updatetime', default=True,
        help='Do not use the Last-modified header to set the file modification time')
    filesystem.add_option(
        '--write-description',
        action='store_true', dest='writedescription', default=False,
        help='Write video description to a .description file')
    filesystem.add_option(
        '--write-info-json',
        action='store_true', dest='writeinfojson', default=False,
        help='Write video metadata to a .info.json file')
    filesystem.add_option(
        '--write-annotations',
        action='store_true', dest='writeannotations', default=False,
        help='Write video annotations to a .annotations.xml file')
    filesystem.add_option(
        '--load-info-json', '--load-info',
        dest='load_info_filename', metavar='FILE',
        help='JSON file containing the video information (created with the "--write-info-json" option)')
    filesystem.add_option(
        '--cookies',
        dest='cookiefile', metavar='FILE',
        help='File to read cookies from and dump cookie jar in')
    filesystem.add_option(
        '--cache-dir', dest='cachedir', default=None, metavar='DIR',
        help='Location in the filesystem where youtube-dl can store some downloaded information permanently. By default $XDG_CACHE_HOME/youtube-dl or ~/.cache/youtube-dl . At the moment, only YouTube player files (for videos with obfuscated signatures) are cached, but that may change.')
    filesystem.add_option(
        '--no-cache-dir', action='store_const', const=False, dest='cachedir',
        help='Disable filesystem caching')
    filesystem.add_option(
        '--rm-cache-dir',
        action='store_true', dest='rm_cachedir',
        help='Delete all filesystem cache files')

    thumbnail = optparse.OptionGroup(parser, 'Thumbnail images')
    thumbnail.add_option(
        '--write-thumbnail',
        action='store_true', dest='writethumbnail', default=False,
        help='Write thumbnail image to disk')
    thumbnail.add_option(
        '--write-all-thumbnails',
        action='store_true', dest='write_all_thumbnails', default=False,
        help='Write all thumbnail image formats to disk')
    thumbnail.add_option(
        '--list-thumbnails',
        action='store_true', dest='list_thumbnails', default=False,
        help='Simulate and list all available thumbnail formats')

    postproc = optparse.OptionGroup(parser, 'Post-processing Options')
    postproc.add_option(
        '-x', '--extract-audio',
        action='store_true', dest='extractaudio', default=False,
        help='Convert video files to audio-only files (requires ffmpeg or avconv and ffprobe or avprobe)')
    postproc.add_option(
        '--audio-format', metavar='FORMAT', dest='audioformat', default='best',
        help='Specify audio format: "best", "aac", "flac", "mp3", "m4a", "opus", "vorbis", or "wav"; "%default" by default; No effect without -x')
    postproc.add_option(
        '--audio-quality', metavar='QUALITY',
        dest='audioquality', default='5',
        help='Specify ffmpeg/avconv audio quality, insert a value between 0 (better) and 9 (worse) for VBR or a specific bitrate like 128K (default %default)')
    postproc.add_option(
        '--recode-video',
        metavar='FORMAT', dest='recodevideo', default=None,
        help='Encode the video to another format if necessary (currently supported: mp4|flv|ogg|webm|mkv|avi)')
    postproc.add_option(
        '--postprocessor-args',
        dest='postprocessor_args', metavar='ARGS',
        help='Give these arguments to the postprocessor')
    postproc.add_option(
        '-k', '--keep-video',
        action='store_true', dest='keepvideo', default=False,
        help='Keep the video file on disk after the post-processing; the video is erased by default')
    postproc.add_option(
        '--no-post-overwrites',
        action='store_true', dest='nopostoverwrites', default=False,
        help='Do not overwrite post-processed files; the post-processed files are overwritten by default')
    postproc.add_option(
        '--embed-subs',
        action='store_true', dest='embedsubtitles', default=False,
        help='Embed subtitles in the video (only for mp4, webm and mkv videos)')
    postproc.add_option(
        '--embed-thumbnail',
        action='store_true', dest='embedthumbnail', default=False,
        help='Embed thumbnail in the audio as cover art')
    postproc.add_option(
        '--add-metadata',
        action='store_true', dest='addmetadata', default=False,
        help='Write metadata to the video file')
    postproc.add_option(
        '--metadata-from-title',
        metavar='FORMAT', dest='metafromtitle',
        help='Parse additional metadata like song title / artist from the video title. '
             'The format syntax is the same as --output. Regular expression with '
             'named capture groups may also be used. '
             'The parsed parameters replace existing values. '
             'Example: --metadata-from-title "%(artist)s - %(title)s" matches a title like '
             '"Coldplay - Paradise". '
             'Example (regex): --metadata-from-title "(?P<artist>.+?) - (?P<title>.+)"')
    postproc.add_option(
        '--xattrs',
        action='store_true', dest='xattrs', default=False,
        help='Write metadata to the video file\'s xattrs (using dublin core and xdg standards)')
    postproc.add_option(
        '--fixup',
        metavar='POLICY', dest='fixup', default='detect_or_warn',
        help='Automatically correct known faults of the file. '
             'One of never (do nothing), warn (only emit a warning), '
             'detect_or_warn (the default; fix file if we can, warn otherwise)')
    postproc.add_option(
        '--prefer-avconv',
        action='store_false', dest='prefer_ffmpeg',
        help='Prefer avconv over ffmpeg for running the postprocessors')
    postproc.add_option(
        '--prefer-ffmpeg',
        action='store_true', dest='prefer_ffmpeg',
        help='Prefer ffmpeg over avconv for running the postprocessors (default)')
    postproc.add_option(
        '--ffmpeg-location', '--avconv-location', metavar='PATH',
        dest='ffmpeg_location',
        help='Location of the ffmpeg/avconv binary; either the path to the binary or its containing directory.')
    postproc.add_option(
        '--exec',
        metavar='CMD', dest='exec_cmd',
        help='Execute a command on the file after downloading and post-processing, similar to find\'s -exec syntax. Example: --exec \'adb push {} /sdcard/Music/ && rm {}\'')
    postproc.add_option(
        '--convert-subs', '--convert-subtitles',
        metavar='FORMAT', dest='convertsubtitles', default=None,
        help='Convert the subtitles to other format (currently supported: srt|ass|vtt|lrc)')

    parser.add_option_group(general)
    parser.add_option_group(network)
    parser.add_option_group(geo)
    parser.add_option_group(selection)
    parser.add_option_group(downloader)
    parser.add_option_group(filesystem)
    parser.add_option_group(thumbnail)
    parser.add_option_group(verbosity)
    parser.add_option_group(workarounds)
    parser.add_option_group(video_format)
    parser.add_option_group(subtitles)
    parser.add_option_group(authentication)
    parser.add_option_group(adobe_pass)
    parser.add_option_group(postproc)

    if overrideArguments is not None:
        opts, args = parser.parse_args(overrideArguments)
        if opts.verbose:
            write_string('[debug] Override config: ' + repr(overrideArguments) + '\n')
    else:
        def compat_conf(conf):
            if sys.version_info < (3,):
                return [a.decode(preferredencoding(), 'replace') for a in conf]
            return conf

        command_line_conf = compat_conf(sys.argv[1:])
        opts, args = parser.parse_args(command_line_conf)

        system_conf = user_conf = custom_conf = []

        if '--config-location' in command_line_conf:
            location = compat_expanduser(opts.config_location)
            if os.path.isdir(location):
                location = os.path.join(location, 'youtube-dl.conf')
            if not os.path.exists(location):
                parser.error('config-location %s does not exist.' % location)
            custom_conf = _readOptions(location)
        elif '--ignore-config' in command_line_conf:
            pass
        else:
            system_conf = _readOptions('/etc/youtube-dl.conf')
            if '--ignore-config' not in system_conf:
                user_conf = _readUserConf()

        argv = system_conf + user_conf + custom_conf + command_line_conf
        opts, args = parser.parse_args(argv)
        if opts.verbose:
            for conf_label, conf in (
                    ('System config', system_conf),
                    ('User config', user_conf),
                    ('Custom config', custom_conf),
                    ('Command-line args', command_line_conf)):
                write_string('[debug] %s: %s\n' % (conf_label, repr(_hide_login_info(conf))))

    return parser, opts, args
# Public Domain SOCKS proxy protocol implementation
# Adapted from https://gist.github.com/bluec0re/cafd3764412967417fd3

from __future__ import unicode_literals

# References:
# SOCKS4 protocol http://www.openssh.com/txt/socks4.protocol
# SOCKS4A protocol http://www.openssh.com/txt/socks4a.protocol
# SOCKS5 protocol https://tools.ietf.org/html/rfc1928
# SOCKS5 username/password authentication https://tools.ietf.org/html/rfc1929

import collections
import socket

from .compat import (
    compat_ord,
    compat_struct_pack,
    compat_struct_unpack,
)

__author__ = 'Timo Schmid <coding@timoschmid.de>'

SOCKS4_VERSION = 4
SOCKS4_REPLY_VERSION = 0x00
# Excerpt from SOCKS4A protocol:
# if the client cannot resolve the destination host's domain name to find its
# IP address, it should set the first three bytes of DSTIP to NULL and the last
# byte to a non-zero value.
SOCKS4_DEFAULT_DSTIP = compat_struct_pack('!BBBB', 0, 0, 0, 0xFF)

SOCKS5_VERSION = 5
SOCKS5_USER_AUTH_VERSION = 0x01
SOCKS5_USER_AUTH_SUCCESS = 0x00


class Socks4Command(object):
    CMD_CONNECT = 0x01
    CMD_BIND = 0x02


class Socks5Command(Socks4Command):
    CMD_UDP_ASSOCIATE = 0x03


class Socks5Auth(object):
    AUTH_NONE = 0x00
    AUTH_GSSAPI = 0x01
    AUTH_USER_PASS = 0x02
    AUTH_NO_ACCEPTABLE = 0xFF  # For server response


class Socks5AddressType(object):
    ATYP_IPV4 = 0x01
    ATYP_DOMAINNAME = 0x03
    ATYP_IPV6 = 0x04


class ProxyError(socket.error):
    ERR_SUCCESS = 0x00

    def __init__(self, code=None, msg=None):
        if code is not None and msg is None:
            msg = self.CODES.get(code) or 'unknown error'
        super(ProxyError, self).__init__(code, msg)


class InvalidVersionError(ProxyError):
    def __init__(self, expected_version, got_version):
        msg = ('Invalid response version from server. Expected {0:02x} got '
               '{1:02x}'.format(expected_version, got_version))
        super(InvalidVersionError, self).__init__(0, msg)


class Socks4Error(ProxyError):
    ERR_SUCCESS = 90

    CODES = {
        91: 'request rejected or failed',
        92: 'request rejected because SOCKS server cannot connect to identd on the client',
        93: 'request rejected because the client program and identd report different user-ids'
    }


class Socks5Error(ProxyError):
    ERR_GENERAL_FAILURE = 0x01

    CODES = {
        0x01: 'general SOCKS server failure',
        0x02: 'connection not allowed by ruleset',
        0x03: 'Network unreachable',
        0x04: 'Host unreachable',
        0x05: 'Connection refused',
        0x06: 'TTL expired',
        0x07: 'Command not supported',
        0x08: 'Address type not supported',
        0xFE: 'unknown username or invalid password',
        0xFF: 'all offered authentication methods were rejected'
    }


class ProxyType(object):
    SOCKS4 = 0
    SOCKS4A = 1
    SOCKS5 = 2


Proxy = collections.namedtuple('Proxy', (
    'type', 'host', 'port', 'username', 'password', 'remote_dns'))


class sockssocket(socket.socket):
    def __init__(self, *args, **kwargs):
        self._proxy = None
        super(sockssocket, self).__init__(*args, **kwargs)

    def setproxy(self, proxytype, addr, port, rdns=True, username=None, password=None):
        assert proxytype in (ProxyType.SOCKS4, ProxyType.SOCKS4A, ProxyType.SOCKS5)

        self._proxy = Proxy(proxytype, addr, port, username, password, rdns)

    def recvall(self, cnt):
        data = b''
        while len(data) < cnt:
            cur = self.recv(cnt - len(data))
            if not cur:
                raise EOFError('{0} bytes missing'.format(cnt - len(data)))
            data += cur
        return data

    def _recv_bytes(self, cnt):
        data = self.recvall(cnt)
        return compat_struct_unpack('!{0}B'.format(cnt), data)

    @staticmethod
    def _len_and_data(data):
        return compat_struct_pack('!B', len(data)) + data

    def _check_response_version(self, expected_version, got_version):
        if got_version != expected_version:
            self.close()
            raise InvalidVersionError(expected_version, got_version)

    def _resolve_address(self, destaddr, default, use_remote_dns):
        try:
            return socket.inet_aton(destaddr)
        except socket.error:
            if use_remote_dns and self._proxy.remote_dns:
                return default
            else:
                return socket.inet_aton(socket.gethostbyname(destaddr))

    def _setup_socks4(self, address, is_4a=False):
        destaddr, port = address

        ipaddr = self._resolve_address(destaddr, SOCKS4_DEFAULT_DSTIP, use_remote_dns=is_4a)

        packet = compat_struct_pack('!BBH', SOCKS4_VERSION, Socks4Command.CMD_CONNECT, port) + ipaddr

        username = (self._proxy.username or '').encode('utf-8')
        packet += username + b'\x00'

        if is_4a and self._proxy.remote_dns:
            packet += destaddr.encode('utf-8') + b'\x00'

        self.sendall(packet)

        version, resp_code, dstport, dsthost = compat_struct_unpack('!BBHI', self.recvall(8))

        self._check_response_version(SOCKS4_REPLY_VERSION, version)

        if resp_code != Socks4Error.ERR_SUCCESS:
            self.close()
            raise Socks4Error(resp_code)

        return (dsthost, dstport)

    def _setup_socks4a(self, address):
        self._setup_socks4(address, is_4a=True)

    def _socks5_auth(self):
        packet = compat_struct_pack('!B', SOCKS5_VERSION)

        auth_methods = [Socks5Auth.AUTH_NONE]
        if self._proxy.username and self._proxy.password:
            auth_methods.append(Socks5Auth.AUTH_USER_PASS)

        packet += compat_struct_pack('!B', len(auth_methods))
        packet += compat_struct_pack('!{0}B'.format(len(auth_methods)), *auth_methods)

        self.sendall(packet)

        version, method = self._recv_bytes(2)

        self._check_response_version(SOCKS5_VERSION, version)

        if method == Socks5Auth.AUTH_NO_ACCEPTABLE or (
                method == Socks5Auth.AUTH_USER_PASS and (not self._proxy.username or not self._proxy.password)):
            self.close()
            raise Socks5Error(Socks5Auth.AUTH_NO_ACCEPTABLE)

        if method == Socks5Auth.AUTH_USER_PASS:
            username = self._proxy.username.encode('utf-8')
            password = self._proxy.password.encode('utf-8')
            packet = compat_struct_pack('!B', SOCKS5_USER_AUTH_VERSION)
            packet += self._len_and_data(username) + self._len_and_data(password)
            self.sendall(packet)

            version, status = self._recv_bytes(2)

            self._check_response_version(SOCKS5_USER_AUTH_VERSION, version)

            if status != SOCKS5_USER_AUTH_SUCCESS:
                self.close()
                raise Socks5Error(Socks5Error.ERR_GENERAL_FAILURE)

    def _setup_socks5(self, address):
        destaddr, port = address

        ipaddr = self._resolve_address(destaddr, None, use_remote_dns=True)

        self._socks5_auth()

        reserved = 0
        packet = compat_struct_pack('!BBB', SOCKS5_VERSION, Socks5Command.CMD_CONNECT, reserved)
        if ipaddr is None:
            destaddr = destaddr.encode('utf-8')
            packet += compat_struct_pack('!B', Socks5AddressType.ATYP_DOMAINNAME)
            packet += self._len_and_data(destaddr)
        else:
            packet += compat_struct_pack('!B', Socks5AddressType.ATYP_IPV4) + ipaddr
        packet += compat_struct_pack('!H', port)

        self.sendall(packet)

        version, status, reserved, atype = self._recv_bytes(4)

        self._check_response_version(SOCKS5_VERSION, version)

        if status != Socks5Error.ERR_SUCCESS:
            self.close()
            raise Socks5Error(status)

        if atype == Socks5AddressType.ATYP_IPV4:
            destaddr = self.recvall(4)
        elif atype == Socks5AddressType.ATYP_DOMAINNAME:
            alen = compat_ord(self.recv(1))
            destaddr = self.recvall(alen)
        elif atype == Socks5AddressType.ATYP_IPV6:
            destaddr = self.recvall(16)
        destport = compat_struct_unpack('!H', self.recvall(2))[0]

        return (destaddr, destport)

    def _make_proxy(self, connect_func, address):
        if not self._proxy:
            return connect_func(self, address)

        result = connect_func(self, (self._proxy.host, self._proxy.port))
        if result != 0 and result is not None:
            return result
        setup_funcs = {
            ProxyType.SOCKS4: self._setup_socks4,
            ProxyType.SOCKS4A: self._setup_socks4a,
            ProxyType.SOCKS5: self._setup_socks5,
        }
        setup_funcs[self._proxy.type](address)
        return result

    def connect(self, address):
        self._make_proxy(socket.socket.connect, address)

    def connect_ex(self, address):
        return self._make_proxy(socket.socket.connect_ex, address)
from __future__ import unicode_literals

import collections
import io
import zlib

from .compat import (
    compat_str,
    compat_struct_unpack,
)
from .utils import (
    ExtractorError,
)


def _extract_tags(file_contents):
    if file_contents[1:3] != b'WS':
        raise ExtractorError(
            'Not an SWF file; header is %r' % file_contents[:3])
    if file_contents[:1] == b'C':
        content = zlib.decompress(file_contents[8:])
    else:
        raise NotImplementedError(
            'Unsupported compression format %r' %
            file_contents[:1])

    # Determine number of bits in framesize rectangle
    framesize_nbits = compat_struct_unpack('!B', content[:1])[0] >> 3
    framesize_len = (5 + 4 * framesize_nbits + 7) // 8

    pos = framesize_len + 2 + 2
    while pos < len(content):
        header16 = compat_struct_unpack('<H', content[pos:pos + 2])[0]
        pos += 2
        tag_code = header16 >> 6
        tag_len = header16 & 0x3f
        if tag_len == 0x3f:
            tag_len = compat_struct_unpack('<I', content[pos:pos + 4])[0]
            pos += 4
        assert pos + tag_len <= len(content), \
            ('Tag %d ends at %d+%d - that\'s longer than the file (%d)'
                % (tag_code, pos, tag_len, len(content)))
        yield (tag_code, content[pos:pos + tag_len])
        pos += tag_len


class _AVMClass_Object(object):
    def __init__(self, avm_class):
        self.avm_class = avm_class

    def __repr__(self):
        return '%s#%x' % (self.avm_class.name, id(self))


class _ScopeDict(dict):
    def __init__(self, avm_class):
        super(_ScopeDict, self).__init__()
        self.avm_class = avm_class

    def __repr__(self):
        return '%s__Scope(%s)' % (
            self.avm_class.name,
            super(_ScopeDict, self).__repr__())


class _AVMClass(object):
    def __init__(self, name_idx, name, static_properties=None):
        self.name_idx = name_idx
        self.name = name
        self.method_names = {}
        self.method_idxs = {}
        self.methods = {}
        self.method_pyfunctions = {}
        self.static_properties = static_properties if static_properties else {}

        self.variables = _ScopeDict(self)
        self.constants = {}

    def make_object(self):
        return _AVMClass_Object(self)

    def __repr__(self):
        return '_AVMClass(%s)' % (self.name)

    def register_methods(self, methods):
        self.method_names.update(methods.items())
        self.method_idxs.update(dict(
            (idx, name)
            for name, idx in methods.items()))


class _Multiname(object):
    def __init__(self, kind):
        self.kind = kind

    def __repr__(self):
        return '[MULTINAME kind: 0x%x]' % self.kind


def _read_int(reader):
    res = 0
    shift = 0
    for _ in range(5):
        buf = reader.read(1)
        assert len(buf) == 1
        b = compat_struct_unpack('<B', buf)[0]
        res = res | ((b & 0x7f) << shift)
        if b & 0x80 == 0:
            break
        shift += 7
    return res


def _u30(reader):
    res = _read_int(reader)
    assert res & 0xf0000000 == 0
    return res


_u32 = _read_int


def _s32(reader):
    v = _read_int(reader)
    if v & 0x80000000 != 0:
        v = - ((v ^ 0xffffffff) + 1)
    return v


def _s24(reader):
    bs = reader.read(3)
    assert len(bs) == 3
    last_byte = b'\xff' if (ord(bs[2:3]) >= 0x80) else b'\x00'
    return compat_struct_unpack('<i', bs + last_byte)[0]


def _read_string(reader):
    slen = _u30(reader)
    resb = reader.read(slen)
    assert len(resb) == slen
    return resb.decode('utf-8')


def _read_bytes(count, reader):
    assert count >= 0
    resb = reader.read(count)
    assert len(resb) == count
    return resb


def _read_byte(reader):
    resb = _read_bytes(1, reader=reader)
    res = compat_struct_unpack('<B', resb)[0]
    return res


StringClass = _AVMClass('(no name idx)', 'String')
ByteArrayClass = _AVMClass('(no name idx)', 'ByteArray')
TimerClass = _AVMClass('(no name idx)', 'Timer')
TimerEventClass = _AVMClass('(no name idx)', 'TimerEvent', {'TIMER': 'timer'})
_builtin_classes = {
    StringClass.name: StringClass,
    ByteArrayClass.name: ByteArrayClass,
    TimerClass.name: TimerClass,
    TimerEventClass.name: TimerEventClass,
}


class _Undefined(object):
    def __bool__(self):
        return False
    __nonzero__ = __bool__

    def __hash__(self):
        return 0

    def __str__(self):
        return 'undefined'
    __repr__ = __str__


undefined = _Undefined()


class SWFInterpreter(object):
    def __init__(self, file_contents):
        self._patched_functions = {
            (TimerClass, 'addEventListener'): lambda params: undefined,
        }
        code_tag = next(tag
                        for tag_code, tag in _extract_tags(file_contents)
                        if tag_code == 82)
        p = code_tag.index(b'\0', 4) + 1
        code_reader = io.BytesIO(code_tag[p:])

        # Parse ABC (AVM2 ByteCode)

        # Define a couple convenience methods
        u30 = lambda *args: _u30(*args, reader=code_reader)
        s32 = lambda *args: _s32(*args, reader=code_reader)
        u32 = lambda *args: _u32(*args, reader=code_reader)
        read_bytes = lambda *args: _read_bytes(*args, reader=code_reader)
        read_byte = lambda *args: _read_byte(*args, reader=code_reader)

        # minor_version + major_version
        read_bytes(2 + 2)

        # Constant pool
        int_count = u30()
        self.constant_ints = [0]
        for _c in range(1, int_count):
            self.constant_ints.append(s32())
        self.constant_uints = [0]
        uint_count = u30()
        for _c in range(1, uint_count):
            self.constant_uints.append(u32())
        double_count = u30()
        read_bytes(max(0, (double_count - 1)) * 8)
        string_count = u30()
        self.constant_strings = ['']
        for _c in range(1, string_count):
            s = _read_string(code_reader)
            self.constant_strings.append(s)
        namespace_count = u30()
        for _c in range(1, namespace_count):
            read_bytes(1)  # kind
            u30()  # name
        ns_set_count = u30()
        for _c in range(1, ns_set_count):
            count = u30()
            for _c2 in range(count):
                u30()
        multiname_count = u30()
        MULTINAME_SIZES = {
            0x07: 2,  # QName
            0x0d: 2,  # QNameA
            0x0f: 1,  # RTQName
            0x10: 1,  # RTQNameA
            0x11: 0,  # RTQNameL
            0x12: 0,  # RTQNameLA
            0x09: 2,  # Multiname
            0x0e: 2,  # MultinameA
            0x1b: 1,  # MultinameL
            0x1c: 1,  # MultinameLA
        }
        self.multinames = ['']
        for _c in range(1, multiname_count):
            kind = u30()
            assert kind in MULTINAME_SIZES, 'Invalid multiname kind %r' % kind
            if kind == 0x07:
                u30()  # namespace_idx
                name_idx = u30()
                self.multinames.append(self.constant_strings[name_idx])
            elif kind == 0x09:
                name_idx = u30()
                u30()
                self.multinames.append(self.constant_strings[name_idx])
            else:
                self.multinames.append(_Multiname(kind))
                for _c2 in range(MULTINAME_SIZES[kind]):
                    u30()

        # Methods
        method_count = u30()
        MethodInfo = collections.namedtuple(
            'MethodInfo',
            ['NEED_ARGUMENTS', 'NEED_REST'])
        method_infos = []
        for method_id in range(method_count):
            param_count = u30()
            u30()  # return type
            for _ in range(param_count):
                u30()  # param type
            u30()  # name index (always 0 for youtube)
            flags = read_byte()
            if flags & 0x08 != 0:
                # Options present
                option_count = u30()
                for c in range(option_count):
                    u30()  # val
                    read_bytes(1)  # kind
            if flags & 0x80 != 0:
                # Param names present
                for _ in range(param_count):
                    u30()  # param name
            mi = MethodInfo(flags & 0x01 != 0, flags & 0x04 != 0)
            method_infos.append(mi)

        # Metadata
        metadata_count = u30()
        for _c in range(metadata_count):
            u30()  # name
            item_count = u30()
            for _c2 in range(item_count):
                u30()  # key
                u30()  # value

        def parse_traits_info():
            trait_name_idx = u30()
            kind_full = read_byte()
            kind = kind_full & 0x0f
            attrs = kind_full >> 4
            methods = {}
            constants = None
            if kind == 0x00:  # Slot
                u30()  # Slot id
                u30()  # type_name_idx
                vindex = u30()
                if vindex != 0:
                    read_byte()  # vkind
            elif kind == 0x06:  # Const
                u30()  # Slot id
                u30()  # type_name_idx
                vindex = u30()
                vkind = 'any'
                if vindex != 0:
                    vkind = read_byte()
                if vkind == 0x03:  # Constant_Int
                    value = self.constant_ints[vindex]
                elif vkind == 0x04:  # Constant_UInt
                    value = self.constant_uints[vindex]
                else:
                    return {}, None  # Ignore silently for now
                constants = {self.multinames[trait_name_idx]: value}
            elif kind in (0x01, 0x02, 0x03):  # Method / Getter / Setter
                u30()  # disp_id
                method_idx = u30()
                methods[self.multinames[trait_name_idx]] = method_idx
            elif kind == 0x04:  # Class
                u30()  # slot_id
                u30()  # classi
            elif kind == 0x05:  # Function
                u30()  # slot_id
                function_idx = u30()
                methods[function_idx] = self.multinames[trait_name_idx]
            else:
                raise ExtractorError('Unsupported trait kind %d' % kind)

            if attrs & 0x4 != 0:  # Metadata present
                metadata_count = u30()
                for _c3 in range(metadata_count):
                    u30()  # metadata index

            return methods, constants

        # Classes
        class_count = u30()
        classes = []
        for class_id in range(class_count):
            name_idx = u30()

            cname = self.multinames[name_idx]
            avm_class = _AVMClass(name_idx, cname)
            classes.append(avm_class)

            u30()  # super_name idx
            flags = read_byte()
            if flags & 0x08 != 0:  # Protected namespace is present
                u30()  # protected_ns_idx
            intrf_count = u30()
            for _c2 in range(intrf_count):
                u30()
            u30()  # iinit
            trait_count = u30()
            for _c2 in range(trait_count):
                trait_methods, trait_constants = parse_traits_info()
                avm_class.register_methods(trait_methods)
                if trait_constants:
                    avm_class.constants.update(trait_constants)

        assert len(classes) == class_count
        self._classes_by_name = dict((c.name, c) for c in classes)

        for avm_class in classes:
            avm_class.cinit_idx = u30()
            trait_count = u30()
            for _c2 in range(trait_count):
                trait_methods, trait_constants = parse_traits_info()
                avm_class.register_methods(trait_methods)
                if trait_constants:
                    avm_class.constants.update(trait_constants)

        # Scripts
        script_count = u30()
        for _c in range(script_count):
            u30()  # init
            trait_count = u30()
            for _c2 in range(trait_count):
                parse_traits_info()

        # Method bodies
        method_body_count = u30()
        Method = collections.namedtuple('Method', ['code', 'local_count'])
        self._all_methods = []
        for _c in range(method_body_count):
            method_idx = u30()
            u30()  # max_stack
            local_count = u30()
            u30()  # init_scope_depth
            u30()  # max_scope_depth
            code_length = u30()
            code = read_bytes(code_length)
            m = Method(code, local_count)
            self._all_methods.append(m)
            for avm_class in classes:
                if method_idx in avm_class.method_idxs:
                    avm_class.methods[avm_class.method_idxs[method_idx]] = m
            exception_count = u30()
            for _c2 in range(exception_count):
                u30()  # from
                u30()  # to
                u30()  # target
                u30()  # exc_type
                u30()  # var_name
            trait_count = u30()
            for _c2 in range(trait_count):
                parse_traits_info()

        assert p + code_reader.tell() == len(code_tag)

    def patch_function(self, avm_class, func_name, f):
        self._patched_functions[(avm_class, func_name)] = f

    def extract_class(self, class_name, call_cinit=True):
        try:
            res = self._classes_by_name[class_name]
        except KeyError:
            raise ExtractorError('Class %r not found' % class_name)

        if call_cinit and hasattr(res, 'cinit_idx'):
            res.register_methods({'$cinit': res.cinit_idx})
            res.methods['$cinit'] = self._all_methods[res.cinit_idx]
            cinit = self.extract_function(res, '$cinit')
            cinit([])

        return res

    def extract_function(self, avm_class, func_name):
        p = self._patched_functions.get((avm_class, func_name))
        if p:
            return p
        if func_name in avm_class.method_pyfunctions:
            return avm_class.method_pyfunctions[func_name]
        if func_name in self._classes_by_name:
            return self._classes_by_name[func_name].make_object()
        if func_name not in avm_class.methods:
            raise ExtractorError('Cannot find function %s.%s' % (
                avm_class.name, func_name))
        m = avm_class.methods[func_name]

        def resfunc