[WARP] Improvements for 5.3

docs/guide/warp.md

@@ -232,6 +232,13 @@ When running the matcher manually, you may get a warning about no relocatable re
 sections or segments in your view. For WARP to work we must have some range of address space to work with, without it the
 function GUIDs will likely be inconsistent if the functions can be based at different addresses.
 
+### "Relocatable region has a low start-address" warning
+
+WARP uses relocatable regions to determine relocatable addresses encoded in instructions, if you have a relocatable region
+that covers a low address space, WARP may mask regular constants and other irrelevant instructions. This warning mostly
+affects firmware binaries (or other mapped views), if you have not rebased the view to the correct image base, then you
+should as it will fix this issue
+
 ### Failed to connect to the server
 
 If you fail to connect to a WARP server, you will receive an error in the log. Outside typical network connectivity issues 

plugins/bntl_utils/src/schema.rs

@@ -5,9 +5,9 @@ use std::path::Path;
 
 #[derive(Deserialize, Debug)]
 pub struct BntlSchema {
-    // The list of library names this library depends on
+    /// The list of library names this library depends on
     pub dependencies: Vec<String>,
-    // Maps internal type IDs or names to their external sources
+    /// Maps internal type IDs or names to their external sources
     pub type_sources: Vec<TypeSource>,
 }
 
@@ -35,8 +35,8 @@ impl BntlSchema {
 
 #[derive(Deserialize, Debug)]
 pub struct TypeSource {
-    // The components of the name, e.g., ["std", "string"]
+    /// The components of the name, e.g., ["std", "string"]
     pub name: Vec<String>,
-    // The name of the dependency library it comes from
+    /// The name of the dependency library it comes from
     pub source: String,
 }

plugins/warp/Cargo.toml

@@ -27,19 +27,11 @@ uuid = { version = "1.12.0", features = ["v4", "serde"] }
 thiserror = "2.0"
 ar = { git = "https://github.com/mdsteele/rust-ar" }
 tempdir = "0.3.7"
-regex = "1.11"
 directories = "6.0"
 compact_str = { version = "0.9.0", features = ["serde"] }
 base64 = "0.22"
 serde_qs = "0.15"
 
-# For reports
-minijinja = "2.10.2"
-minijinja-embed = "2.10.2"
-
-[build-dependencies]
-minijinja-embed = "2.10.2"
-
 [dev-dependencies]
 criterion = "0.6"
 insta = { version = "1.42", features = ["yaml"] }

plugins/warp/api/python/generator.cpp

@@ -321,6 +321,11 @@ int main(int argc, char* argv[])
 			fprintf(out, "from binaryninja._binaryninjacore import BNType, BNTypeHandle\n");
 			continue;
 		}
+		if (name == "BNDataBuffer")
+		{
+			fprintf(out, "from binaryninja._binaryninjacore import BNDataBuffer, BNDataBufferHandle\n");
+			continue;
+		}
 		if (i.second->GetClass() == StructureTypeClass)
 		{
 			fprintf(out, "class %s(ctypes.Structure):\n", name.c_str());

plugins/warp/api/python/warp.py

@@ -4,11 +4,11 @@
 from typing import List, Optional, Union
 
 import binaryninja
-from binaryninja import BinaryView, Function, BasicBlock, Architecture, Platform, Type, Symbol, LowLevelILInstruction, LowLevelILFunction
+from binaryninja import BinaryView, Function, BasicBlock, Architecture, Platform, Type, Symbol, LowLevelILInstruction, LowLevelILFunction, DataBuffer, Project, ProjectFile
 from binaryninja._binaryninjacore import BNFreeString, BNAllocString, BNType
 
 from . import _warpcore as warpcore
-from .warp_enums import WARPContainerSearchItemKind
+from .warp_enums import WARPContainerSearchItemKind, WARPProcessorIncludedData, WARPProcessorIncludedFunctions
 
 
 class WarpUUID:
@@ -74,6 +74,30 @@ def __repr__(self):
         return f"<TypeGUID '{str(self)}'>"
 
 
+class WarpType:
+    def __init__(self, handle: warpcore.BNWARPType):
+        self.handle = handle
+
+    def __del__(self):
+        if self.handle is not None:
+            warpcore.BNWARPFreeTypeReference(self.handle)
+
+    def __repr__(self):
+        return f"<WarpType name: '{self.name}' confidence: '{self.confidence}'>"
+
+    @property
+    def name(self) -> str:
+        return warpcore.BNWARPTypeGetName(self.handle)
+
+    @property
+    def confidence(self) -> int:
+        return warpcore.BNWARPTypeGetConfidence(self.handle)
+
+    def analysis_type(self, arch: Optional[Architecture] = None) -> Type:
+        if arch is None:
+            return Type.create(handle=warpcore.BNWARPTypeGetAnalysisType(None, self.handle))
+        return Type.create(handle=warpcore.BNWARPTypeGetAnalysisType(arch.handle, self.handle))
+
 @dataclasses.dataclass
 class WarpFunctionComment:
     text: str
@@ -155,11 +179,12 @@ def get_symbol(self, function: Function) -> Symbol:
         symbol_handle = warpcore.BNWARPFunctionGetSymbol(self.handle, function.handle)
         return Symbol(symbol_handle)
 
-    def get_type(self, function: Function) -> Optional[Type]:
-        type_handle = warpcore.BNWARPFunctionGetType(self.handle, function.handle)
+    @property
+    def type(self) -> Optional[WarpType]:
+        type_handle = warpcore.BNWARPFunctionGetType(self.handle)
         if not type_handle:
             return None
-        return Type(type_handle)
+        return WarpType(type_handle)
 
     @property
     def constraints(self) -> List[WarpConstraint]:
@@ -259,11 +284,12 @@ def source(self) -> Source:
     def name(self) -> str:
         return warpcore.BNWARPContainerSearchItemGetName(self.handle)
 
-    def get_type(self, arch: Architecture) -> Optional[Type]:
-        ty = warpcore.BNWARPContainerSearchItemGetType(arch.handle, self.handle)
+    @property
+    def type(self) -> Optional[WarpType]:
+        ty = warpcore.BNWARPContainerSearchItemGetType(self.handle)
         if not ty:
             return None
-        return Type(ty)
+        return WarpType(ty)
 
     @property
     def function(self) -> Optional[WarpFunction]:
@@ -405,12 +431,12 @@ def add_functions(self, target: WarpTarget, source: Source, functions: List[Func
             core_funcs[i] = functions[i].handle
         return warpcore.BNWARPContainerAddFunctions(self.handle, target.handle, source.uuid, core_funcs, count)
 
-    def add_types(self, view: BinaryView, source: Source, types: List[Type]) -> bool:
+    def add_types(self, source: Source, types: List[WarpType]) -> bool:
         count = len(types)
-        core_types = (ctypes.POINTER(BNType) * count)()
+        core_types = (ctypes.POINTER(warpcore.BNWARPType) * count)()
         for i in range(count):
             core_types[i] = types[i].handle
-        return warpcore.BNWARPContainerAddTypes(view.handle, self.handle, source.uuid, core_types, count)
+        return warpcore.BNWARPContainerAddTypes(self.handle, source.uuid, core_types, count)
 
     def remove_functions(self, target: WarpTarget, source: Source, functions: List[Function]) -> bool:
         count = len(functions)
@@ -426,19 +452,25 @@ def remove_types(self, source: Source, guids: List[TypeGUID]) -> bool:
             core_guids[i] = guids[i].uuid
         return warpcore.BNWARPContainerRemoveTypes(self.handle, source.uuid, core_guids, count)
 
-    def fetch_functions(self, target: WarpTarget, guids: List[FunctionGUID], source_tags: Optional[List[str]] = None):
+    def fetch_functions(self, target: WarpTarget, guids: List[FunctionGUID], source_tags: Optional[List[str]] = None, constraints: Optional[List[ConstraintGUID]] = None):
         count = len(guids)
         core_guids = (warpcore.BNWARPFunctionGUID * count)()
         for i in range(count):
             core_guids[i] = guids[i].uuid
+        if constraints is None:
+            constraints = []
+        constraints_count = len(constraints)
+        core_constraints = (warpcore.BNWARPConstraintGUID * constraints_count)()
+        for i in range(constraints_count):
+            core_constraints[i] = constraints[i].uuid
         if source_tags is None:
             source_tags = []
         source_tags_ptr = (ctypes.c_char_p * len(source_tags))()
         source_tags_len = len(source_tags)
         for i in range(len(source_tags)):
             source_tags_ptr[i] = source_tags[i].encode('utf-8')
         source_tags_array_ptr = ctypes.cast(source_tags_ptr, ctypes.POINTER(ctypes.c_char_p))
-        warpcore.BNWARPContainerFetchFunctions(self.handle, target.handle, source_tags_array_ptr, source_tags_len, core_guids, count)
+        warpcore.BNWARPContainerFetchFunctions(self.handle, target.handle, source_tags_array_ptr, source_tags_len, core_guids, count, core_constraints, constraints_count)
 
     def get_sources_with_function_guid(self, target: WarpTarget, guid: FunctionGUID) -> List[Source]:
         count = ctypes.c_size_t()
@@ -473,11 +505,11 @@ def get_functions_with_guid(self, target: WarpTarget, source: Source, guid: Func
         warpcore.BNWARPFreeFunctionList(funcs, count.value)
         return result
 
-    def get_type_with_guid(self, arch: Architecture, source: Source, guid: TypeGUID) -> Optional[Type]:
-        ty = warpcore.BNWARPContainerGetTypeWithGUID(arch.handle, self.handle, source.uuid, guid.uuid)
+    def get_type_with_guid(self, source: Source, guid: TypeGUID) -> Optional[WarpType]:
+        ty = warpcore.BNWARPContainerGetTypeWithGUID(self.handle, source.uuid, guid.uuid)
         if not ty:
             return None
-        return Type(ty)
+        return WarpType(ty)
 
     def get_type_guids_with_name(self, source: Source, name: str) -> List[TypeGUID]:
         count = ctypes.c_size_t()
@@ -497,6 +529,128 @@ def search(self, query: WarpContainerSearchQuery) -> Optional[WarpContainerRespo
         return WarpContainerResponse.from_api(response.contents)
 
 
+class WarpChunk:
+    def __init__(self, handle: warpcore.BNWARPChunk):
+        self.handle = handle
+
+    def __del__(self):
+        if self.handle is not None:
+            warpcore.BNWARPFreeChunkReference(self.handle)
+
+    def __repr__(self):
+        return f"<WarpChunk functions: '{len(self.functions)}' types: '{len(self.types)}'>"
+
+    @property
+    def functions(self) -> List[WarpFunction]:
+        count = ctypes.c_size_t()
+        funcs = warpcore.BNWARPChunkGetFunctions(self.handle, count)
+        if not funcs:
+            return []
+        result = []
+        for i in range(count.value):
+            result.append(WarpFunction(warpcore.BNWARPNewFunctionReference(funcs[i])))
+        warpcore.BNWARPFreeFunctionList(funcs, count.value)
+        return result
+
+    @property
+    def types(self) -> List[WarpType]:
+        count = ctypes.c_size_t()
+        types = warpcore.BNWARPChunkGetTypes(self.handle, count)
+        if not types:
+            return []
+        result = []
+        for i in range(count.value):
+            result.append(WarpType(warpcore.BNWARPNewTypeReference(types[i])))
+        warpcore.BNWARPFreeTypeList(types, count.value)
+        return result
+
+class WarpFile:
+    def __init__(self, handle: Union[warpcore.BNWARPFileHandle, str]):
+        if isinstance(handle, str):
+            self.handle = warpcore.BNWARPNewFileFromPath(handle)
+        else:
+            self.handle = handle
+
+    def __del__(self):
+        if self.handle is not None:
+            warpcore.BNWARPFreeFileReference(self.handle)
+
+    def __repr__(self):
+        return f"<WarpFile chunks: '{len(self.chunks)}'>"
+
+    @property
+    def chunks(self) -> List[WarpChunk]:
+        count = ctypes.c_size_t()
+        chunks = warpcore.BNWARPFileGetChunks(self.handle, count)
+        if not chunks:
+            return []
+        result = []
+        for i in range(count.value):
+            result.append(WarpChunk(warpcore.BNWARPNewChunkReference(chunks[i])))
+        warpcore.BNWARPFreeChunkList(chunks, count.value)
+        return result
+
+    def to_data_buffer(self) -> DataBuffer:
+        return DataBuffer(handle=warpcore.BNWARPFileToDataBuffer(self.handle))
+
+
+@dataclasses.dataclass
+class WarpProcessorState:
+    cancelled: bool = False
+    unprocessed_file_count: int = 0
+    processed_file_count: int = 0
+    analyzing_files: List[str] = dataclasses.field(default_factory=list)
+    processing_files: List[str] = dataclasses.field(default_factory=list)
+
+    @staticmethod
+    def from_api(state: warpcore.BNWARPProcessorState) -> 'WarpProcessorState':
+        analyzing_files = []
+        processing_files = []
+        for i in range(state.analyzing_files_count):
+            analyzing_files.append(state.analyzing_files[i])
+        for i in range(state.processing_files_count):
+            processing_files.append(state.processing_files[i])
+        return WarpProcessorState(
+            cancelled=state.cancelled,
+            unprocessed_file_count=state.unprocessed_file_count,
+            processed_file_count=state.processed_file_count,
+            analyzing_files=analyzing_files,
+            processing_files=processing_files
+        )
+
+class WarpProcessor:
+    def __init__(self, included_data: WARPProcessorIncludedData = WARPProcessorIncludedData.WARPProcessorIncludedDataAll,
+                 included_functions: WARPProcessorIncludedFunctions = WARPProcessorIncludedFunctions.WARPProcessorIncludedFunctionsAnnotated,
+                 worker_count: int = 1):
+        self.handle = warpcore.BNWARPNewProcessor(ctypes.c_int(included_data), ctypes.c_int(included_functions), worker_count)
+
+    def __del__(self):
+        if self.handle is not None:
+            warpcore.BNWARPFreeProcessor(self.handle)
+
+    def add_path(self, path: str):
+        warpcore.BNWARPProcessorAddPath(self.handle, path)
+
+    def add_project(self, project: Project):
+        warpcore.BNWARPProcessorAddProject(self.handle, project.handle)
+
+    def add_project_file(self, project_file: ProjectFile):
+        warpcore.BNWARPProcessorAddProjectFile(self.handle, project_file.handle)
+
+    def add_binary_view(self, view: BinaryView):
+        warpcore.BNWARPProcessorAddBinaryView(self.handle, view.handle)
+
+    def start(self) -> Optional[WarpFile]:
+        file = warpcore.BNWARPProcessorStart(self.handle)
+        if not file:
+            return None
+        return WarpFile(file)
+
+    def state(self) -> WarpProcessorState:
+        state_raw = warpcore.BNWARPProcessorGetState(self.handle)
+        warpcore.BNWARPFreeProcessorState(state_raw)
+        return WarpProcessorState.from_api(state_raw)
+
 def run_matcher(view: BinaryView):
     warpcore.BNWARPRunMatcher(view.handle)
 

plugins/warp/api/python/warp_enums.py

@@ -6,3 +6,16 @@ class WARPContainerSearchItemKind(enum.IntEnum):
 	WARPContainerSearchItemKindFunction = 1
 	WARPContainerSearchItemKindType = 2
 	WARPContainerSearchItemKindSymbol = 3
+
+
+class WARPProcessorIncludedData(enum.IntEnum):
+	WARPProcessorIncludedDataSymbols = 0
+	WARPProcessorIncludedDataSignatures = 1
+	WARPProcessorIncludedDataTypes = 2
+	WARPProcessorIncludedDataAll = 3
+
+
+class WARPProcessorIncludedFunctions(enum.IntEnum):
+	WARPProcessorIncludedFunctionsSelected = 0
+	WARPProcessorIncludedFunctionsAnnotated = 1
+	WARPProcessorIncludedFunctionsAll = 2