Search files using the fastest Regex Engine ever - ripgrep - replacement is also supported!

Uses https://github.com/BurntSushi/ripgrep and parses the regular expression output into a pandas DataFrame.

It is much faster than the first version: https://github.com/hansalemaos/PYRipGREP due to more efficient parsing, and substitution is

also supported (BACK UP YOUR DATA BEFORE!). Check out the examples

Tested against Windows 10 / Python 3.10 / Anaconda

pip install rushex

import random



from rushex import FullBore





monsterregex = FullBore(ripgrepexe=r"rg.exe") 





files = [

    r"C:\testxt\xab.txt",

    r"C:\testxt\ö.txt",

    r"C:\testxt\xaa.txt",

    r"C:\testxt\xaa2.txt",

    r"C:\testxt\pypigitupload.py",

    r"C:\testxt\pyripbro.py",

    r"C:\testxt\pip2dfxxxxxxxxx.py",

]

dfa = monsterregex.find_all_in_files(

    regular_expressions=[

        r"\b\w{15}\b",

        r"\b\d{4}\b",

    ],  # pass as many regular expressions as you want

    files=files,

    ignore_case=True,

    allowed_extensions=(),  # if empty, no file ending is excluded

    binary=True,

    dfa_size="1G",

    multiline=False,

    multiline_dotall=False,

    field_match_separator="ÇÇÇÇÇ",

)  # A string which won't show up in your search results



# print(dfa[:5].to_string())

#                     aa_filename  aa_line  aa_byte_offset_o     aa_string_bytes  aa_byte_len        aa_string aa_replacement                aa_newfilename

# 0  C:\testxt\pip2dfxxxxxxxxx.py        1               589  b'ignore_packages'           15  ignore_packages            b''  C:\testxt\pip2dfxxxxxxxxx.py

# 1  C:\testxt\pip2dfxxxxxxxxx.py        1               650  b'ignore_packages'           15  ignore_packages            b''  C:\testxt\pip2dfxxxxxxxxx.py

# 2  C:\testxt\pip2dfxxxxxxxxx.py        1               694  b'ignore_packages'           15  ignore_packages            b''  C:\testxt\pip2dfxxxxxxxxx.py

# 3  C:\testxt\pip2dfxxxxxxxxx.py        1               731  b'ignore_packages'           15  ignore_packages            b''  C:\testxt\pip2dfxxxxxxxxx.py

# 4  C:\testxt\pip2dfxxxxxxxxx.py        1              1126  b'ignore_packages'           15  ignore_packages            b''  C:\testxt\pip2dfxxxxxxxxx.py



df3 = monsterregex.find_all_in_folders(

    folders=[r"C:\ProgramData\anaconda3\envs\adda"],

    regular_expressions=[r"\bnumexpr|pandas\b", r"\bnp\b."],

    allowed_extensions=(".py",),  # searches only in py files

    ignore_case=True,

    maxsubfolders=-1,  # all subfolders

)



# print(df3[:5].to_string())

#                                                  aa_filename  aa_line  aa_byte_offset_o aa_string_bytes  aa_byte_len aa_string aa_replacement                                             aa_newfilename

# 0  C:\ProgramData\anaconda3\envs\adda\winrtocrxxxxxxxxxxx.py        1                 7       b'pandas'            6    pandas            b''  C:\ProgramData\anaconda3\envs\adda\winrtocrxxxxxxxxxxx.py

# 1  C:\ProgramData\anaconda3\envs\adda\winrtocrxxxxxxxxxxx.py        1               135       b'pandas'            6    pandas            b''  C:\ProgramData\anaconda3\envs\adda\winrtocrxxxxxxxxxxx.py

# 2           C:\ProgramData\anaconda3\envs\adda\rushextest.py        1              1173       b'pandas'            6    pandas            b''           C:\ProgramData\anaconda3\envs\adda\rushextest.py

# 3           C:\ProgramData\anaconda3\envs\adda\rushextest.py        1              1433       b'pandas'            6    pandas            b''           C:\ProgramData\anaconda3\envs\adda\rushextest.py

# 4           C:\ProgramData\anaconda3\envs\adda\rushextest.py        1              1690       b'pandas'            6    pandas            b''           C:\ProgramData\anaconda3\envs\adda\rushextest.py

df4 = monsterregex.find_all_in_folders(

    folders=[r"C:\ProgramData\anaconda3\envs\adda"],

    regular_expressions=[r"\bnumexpr|pandas\b", r"\bnp\.\b"],

    allowed_extensions=(),

    ignore_case=True,

    maxsubfolders=-1,

)

#                                                                 aa_filename  aa_line  aa_byte_offset_o aa_string_bytes  aa_byte_len aa_string aa_replacement                                                            aa_newfilename

# 0       C:\ProgramData\anaconda3\envs\adda\__pycache__\wocr.cpython-310.pyc      222               904       b'pandas'            6    pandas            b''       C:\ProgramData\anaconda3\envs\adda\__pycache__\wocr.cpython-310.pyc

# 1  C:\ProgramData\anaconda3\envs\adda\__pycache__\pdmemedit.cpython-310.pyc      515              3496          b'np.'            3       np.            b''  C:\ProgramData\anaconda3\envs\adda\__pycache__\pdmemedit.cpython-310.pyc

# 2  C:\ProgramData\anaconda3\envs\adda\__pycache__\pdmemedit.cpython-310.pyc     1318              8867       b'pandas'            6    pandas            b''  C:\ProgramData\anaconda3\envs\adda\__pycache__\pdmemedit.cpython-310.pyc

# 3  C:\ProgramData\anaconda3\envs\adda\__pycache__\pdmemedit.cpython-310.pyc     1318              9327       b'pandas'            6    pandas            b''  C:\ProgramData\anaconda3\envs\adda\__pycache__\pdmemedit.cpython-310.pyc

# 4  C:\ProgramData\anaconda3\envs\adda\__pycache__\pdmemedit.cpython-310.pyc     2827             15327      b'numexpr'            7   numexpr            b''  C:\ProgramData\anaconda3\envs\adda\__pycache__\pdmemedit.cpython-310.pyc

df5 = monsterregex.find_all_in_folders(

    folders=[r"C:\ProgramData\anaconda3\envs\adda"],

    regular_expressions=[r"\bnumexpr|pandas\b", r"\bnp\b."],

    allowed_extensions=(".py",),

    ignore_case=True,

    maxsubfolders=2,

    binary=False,

    dfa_size="100M",

    multiline=False,

    multiline_dotall=False,

    field_match_separator="ÇÇÇÇÇ",

)

#                                                  aa_filename  aa_line  aa_byte_offset_o aa_string_bytes  aa_byte_len aa_string aa_replacement                                             aa_newfilename

# 0  C:\ProgramData\anaconda3\envs\adda\winrtocrxxxxxxxxxxx.py        1                 7       b'pandas'            6    pandas            b''  C:\ProgramData\anaconda3\envs\adda\winrtocrxxxxxxxxxxx.py

# 1  C:\ProgramData\anaconda3\envs\adda\winrtocrxxxxxxxxxxx.py        1               135       b'pandas'            6    pandas            b''  C:\ProgramData\anaconda3\envs\adda\winrtocrxxxxxxxxxxx.py

# 2           C:\ProgramData\anaconda3\envs\adda\rushextest.py        1              1173       b'pandas'            6    pandas            b''           C:\ProgramData\anaconda3\envs\adda\rushextest.py

# 3           C:\ProgramData\anaconda3\envs\adda\rushextest.py        1              1433       b'pandas'            6    pandas            b''           C:\ProgramData\anaconda3\envs\adda\rushextest.py

# 4           C:\ProgramData\anaconda3\envs\adda\rushextest.py        1              1690       b'pandas'            6    pandas            b''           C:\ProgramData\anaconda3\envs\adda\rushextest.py

df6 = monsterregex.find_all_in_folders(

    folders=[r"C:\ProgramData\anaconda3\envs\adda"],

    regular_expressions=[r"\bnp\b.*?\bpd\b"],

    allowed_extensions=(".py",),

    ignore_case=True,

    maxsubfolders=-1,

    binary=False,

    dfa_size="100M",

    multiline=True,

    multiline_dotall=True,

    field_match_separator="ÇÇÇÇÇ",

)

# print(df5[:6].to_string())

#                                                  aa_filename  aa_line  aa_byte_offset_o aa_string_bytes  aa_byte_len aa_string aa_replacement                                             aa_newfilename

# 0  C:\ProgramData\anaconda3\envs\adda\winrtocrxxxxxxxxxxx.py        1                 7       b'pandas'            6    pandas            b''  C:\ProgramData\anaconda3\envs\adda\winrtocrxxxxxxxxxxx.py

# 1  C:\ProgramData\anaconda3\envs\adda\winrtocrxxxxxxxxxxx.py        1               135       b'pandas'            6    pandas            b''  C:\ProgramData\anaconda3\envs\adda\winrtocrxxxxxxxxxxx.py

# 2           C:\ProgramData\anaconda3\envs\adda\rushextest.py        1              1173       b'pandas'            6    pandas            b''           C:\ProgramData\anaconda3\envs\adda\rushextest.py

# 3           C:\ProgramData\anaconda3\envs\adda\rushextest.py        1              1433       b'pandas'            6    pandas            b''           C:\ProgramData\anaconda3\envs\adda\rushextest.py

# 4           C:\ProgramData\anaconda3\envs\adda\rushextest.py        1              1690       b'pandas'            6    pandas            b''           C:\ProgramData\anaconda3\envs\adda\rushextest.py

# 5           C:\ProgramData\anaconda3\envs\adda\rushextest.py        1              2528       b'pandas'            6    pandas            b''           C:\ProgramData\anaconda3\envs\adda\rushextest.py





# You can search and replace with rushex



# Get a DataFrame

df7 = monsterregex.find_all_in_folders(

    folders=[r"C:\grepte\homepagesavetest13", r"C:\grepte\homepagesavetest14"],

    regular_expressions=[r"der|die|das"], # Pass only one regular expression when you want to edit files!!! More than one might corrupt your data!

    allowed_extensions=(".txt", ".html"), 

    ignore_case=True,

    maxsubfolders=-1,

    binary=True,

    dfa_size="100M",

    multiline=True,

    multiline_dotall=False,

    field_match_separator="ÇÇÇÇÇ",

)



# In the DataFrame, there are 2 columns for configuring the replacement: [aa_replacement, aa_newfilename]

# The column aa_replacement MUST contain binary data. I created a method to make your life a little easier: FullBore.cb (not mandatory)

# The column aa_newfilename MUST contain the new file name, if the folders don't exist yet, they will be created.

df7.aa_replacement = df7.apply(  # Different dtypes to bytes

    lambda x: FullBore.cb(100)

    + FullBore.cb(random.choice(["Ü", "Ä", "Ö"]))

    + FullBore.cb("-------------")

    + FullBore.cb(x.aa_string_bytes[0]).upper()

    + FullBore.cb(x.aa_string_bytes[1]).lower()

    + FullBore.cb(x.aa_string_bytes[-1]).upper()

    + b"xxxx"

    + FullBore.cb(

        FullBore.cb(str(random.randint(1, 2000000))),

    ),

    axis=1,

)



# This is how the DataFrame looks like when it is ready:

# print(df7[:6].to_string())

#                                                                        aa_filename  aa_line  aa_byte_offset_o aa_string_bytes  aa_byte_len aa_string                           aa_replacement                                                                   aa_newfilename

# 0  C:\grepte\homepagesavetest13\www.duden.de\rechtschreibung\Seniorenwohnheim.html        1              1570          b'DeR'            3       DeR   b'd\xc3\x96-------------DeRxxxx727601'  C:\grepte\homepagesavetest13\www.duden.de\rechtschreibung\Seniorenwohnheim.html

# 1  C:\grepte\homepagesavetest13\www.duden.de\rechtschreibung\Seniorenwohnheim.html        1              2116          b'DeR'            3       DeR  b'd\xc3\x84-------------DeRxxxx1859128'  C:\grepte\homepagesavetest13\www.duden.de\rechtschreibung\Seniorenwohnheim.html

# 2  C:\grepte\homepagesavetest13\www.duden.de\rechtschreibung\Seniorenwohnheim.html        1              2703          b'DeR'            3       DeR   b'd\xc3\x9c-------------DeRxxxx460032'  C:\grepte\homepagesavetest13\www.duden.de\rechtschreibung\Seniorenwohnheim.html

# 3  C:\grepte\homepagesavetest13\www.duden.de\rechtschreibung\Seniorenwohnheim.html        1              5278          b'DeR'            3       DeR  b'd\xc3\x84-------------DeRxxxx1014873'  C:\grepte\homepagesavetest13\www.duden.de\rechtschreibung\Seniorenwohnheim.html

# 4  C:\grepte\homepagesavetest13\www.duden.de\rechtschreibung\Seniorenwohnheim.html        1             18775          b'DeR'            3       DeR   b'd\xc3\x96-------------DeRxxxx335438'  C:\grepte\homepagesavetest13\www.duden.de\rechtschreibung\Seniorenwohnheim.html

# 5  C:\grepte\homepagesavetest13\www.duden.de\rechtschreibung\Seniorenwohnheim.html        1             26052          b'DeR'            3       DeR  b'd\xc3\x84-------------DeRxxxx1839084'  C:\grepte\homepagesavetest13\www.duden.de\rechtschreibung\Seniorenwohnheim.html



# Now you can run the replacement, dryrun defaults True, so it won't do anything to your files yet. Disable dryrun to

# get it gone, but MAKE A BACKUP BEFORE!

monsterregex.sub(df7)

# DRY RUN:

# C:\grepte\homepagesavetest14\www.duden.de\synonyme\fortwuchern.html

# OLD: 	b'DaS'

# NEW: 	b'd\xc3\x9c-------------DaSxxxx981614'

# ----------------------

# DRY RUN:

# C:\grepte\homepagesavetest14\www.duden.de\synonyme\fortwuchern.html

# OLD: 	b'DiE'

# NEW: 	b'd\xc3\x84-------------DiExxxx303843'

# ----------------------

# DRY RUN:

# C:\grepte\homepagesavetest14\www.duden.de\synonyme\fortwuchern.html

# OLD: 	b'DeR'

# NEW: 	b'd\xc3\x9c-------------DeRxxxx1319395'

monsterregex.sub(df7, dryrun=False)



# You can do the same thing with Python variables, but it is much slower than the file based version.

# I recommend using the module "regex" https://pypi.org/project/regex/ when working with Python strings and bytes.



with open(r"C:\all_corpora_filtered_maryfied.txt", mode="rb") as f:

    data = f.read()



datau = data.decode("utf-8", "ignore")





results1 = monsterregex.find_all_in_variable(

    regular_expressions=[r"\bHaus\w+\b"], variable=data

)

class FullBore(builtins.object)

 |  FullBore(ripgrepexe: Optional[str] = None, msvc_or_gnu='msvc')

 |  

 |  Methods defined here:

 |  

 |  __init__(self, ripgrepexe: Optional[str] = None, msvc_or_gnu='msvc')

 |      Initializes the FullBore class with the path to the ripgrep executable.

 |      

 |      Args:

 |          ripgrepexe (Union[str, None], optional): Path to the ripgrep executable. Defaults to None.

 |          msvc_or_gnu (str, optional): Compiler to use for installing ripgrep. Defaults to 'msvc'.

 |  

 |  find_all_in_files(self, regular_expressions: Union[list, str], files: Union[list, str], ignore_case: bool = True, allowed_extensions: tuple = (), binary: bool = True, dfa_size: str = '1G', multiline: bool = False, multiline_dotall: bool = False, field_match_separator: str = 'ÇÇÇÇÇ') -> pandas.core.frame.DataFrame

 |      Searches for all regular expressions in the input files.

 |      

 |      Args:

 |          regular_expressions (Union[list, str]): Regular expressions to be searched.

 |          files (Union[list, str]): Input files to be searched.

 |          ignore_case (bool, optional): Ignore case while searching. Defaults to True.

 |          allowed_extensions (tuple, optional): Allowed file extensions. Defaults to ().

 |          binary (bool, optional): Search in binary mode. Defaults to True.

 |          dfa_size (str, optional): DFA size for ripgrep. Defaults to "1G".

 |          multiline (bool, optional): Search in multiline mode. Defaults to False.

 |          multiline_dotall (bool, optional): Search in multiline dotall mode. Defaults to False.

 |          field_match_separator (str, optional): Field match separator. Defaults to "ÇÇÇÇÇ".

 |      

 |      Returns:

 |          pd.DataFrame: Dataframe containing the search results.

 |  

 |  find_all_in_folders(self, regular_expressions: Union[list, str], folders: Union[list, str], ignore_case: bool = True, allowed_extensions: tuple = (), maxsubfolders: int = -1, binary: bool = True, dfa_size: str = '1G', multiline: bool = False, multiline_dotall: bool = False, field_match_separator: str = 'ÇÇÇÇÇ') -> pandas.core.frame.DataFrame

 |      Searches for all regular expressions in the input folders.

 |      

 |      Args:

 |          regular_expressions (Union[list, str]): Regular expressions to be searched.

 |          folders (Union[list, str]): Input folders to be searched.

 |          ignore_case (bool, optional): Ignore case while searching. Defaults to True.

 |          allowed_extensions (tuple, optional): Allowed file extensions. Defaults to ().

 |          maxsubfolders (int, optional): Maximum number of subfolders to be searched. Defaults to -1.

 |          binary (bool, optional): Search in binary mode. Defaults to True.

 |          dfa_size (str, optional): DFA size for ripgrep. Defaults to "1G".

 |          multiline (bool, optional): Search in multiline mode. Defaults to False.

 |          multiline_dotall (bool, optional): Search in multiline dotall mode. Defaults to False.

 |          field_match_separator (str, optional): Field match separator. Defaults to "ÇÇÇÇÇ".

 |      

 |      Returns:

 |          pd.DataFrame: Dataframe containing the search results.

 |  

 |  find_all_in_variable(self, regular_expressions: Union[list, str], variable: Union[bytes, str], ignore_case: bool = True, binary: bool = True, dfa_size: str = '1G', multiline: bool = False, multiline_dotall: bool = False, outputencoding: str = 'utf-8', field_match_separator: str = 'ÇÇÇÇÇ') -> pandas.core.frame.DataFrame

 |      Searches for all regular expressions in the input variable.

 |      

 |      Args:

 |          regular_expressions (Union[list, str]): Regular expressions to be searched.

 |          variable (Union[bytes, str]): Input variable to be searched.

 |          ignore_case (bool, optional): Ignore case while searching. Defaults to True.

 |          binary (bool, optional): Search in binary mode. Defaults to True.

 |          dfa_size (str, optional): DFA size for ripgrep. Defaults to "1G".

 |          multiline (bool, optional): Search in multiline mode. Defaults to False.

 |          multiline_dotall (bool, optional): Search in multiline dotall mode. Defaults to False.

 |          outputencoding (str, optional): Output encoding. Defaults to "utf-8".

 |          field_match_separator (str, optional): Field match separator. Defaults to "ÇÇÇÇÇ".

 |      

 |      Returns:

 |          pd.DataFrame: Dataframe containing the search results.

 |  

 |  sub(self, df: pandas.core.frame.DataFrame, dryrun: bool = True) -> list

 |      Substitutes the matched regular expressions in the input files.

 |      

 |      Args:

 |          df (pd.DataFrame): Dataframe containing the search results.

 |          dryrun (bool, optional): If True, performs a dry run. Defaults to True.

 |      

 |      Returns:

 |          list: List of files where the substitutions were made.

 |  

 |  sub_in_variable(self, df: pandas.core.frame.DataFrame, variable: Union[bytes, str]) -> Union[bytes, str]

 |      Substitutes the matched regular expressions in the input variable.

 |      

 |      Args:

 |          df (pd.DataFrame): Dataframe containing the search/replace results.

 |          variable (Union[bytes, str]): Input variable to be changed.

 |      

 |      Returns:

 |          Union[bytes, str]: Substituted variable.

 |  

 |  ----------------------------------------------------------------------

 |  Static methods defined here:

 |  

 |  cb(variable: Union[str, int, bytes]) -> bytes

 |      Converts the input variable to bytes.

 |      

 |      Args:

 |          variable (Union[str, int, bytes]): Input variable to be converted.

 |      

 |      Returns:

 |          bytes: Converted variable in bytes.

 |