Initial import: grid-bot — grid trading bot for BTC-USDT on Cifra Markets
This commit is contained in:
Kolp
+173
@@ -0,0 +1,173 @@
|
||||
#!/usr/bin/env/python3
|
||||
|
||||
# Copyright (c) Facebook, Inc. and its affiliates.
|
||||
# All rights reserved.
|
||||
#
|
||||
# This source code is licensed under the BSD-style license found in the
|
||||
# LICENSE file in the root directory of this source tree.
|
||||
|
||||
"""
|
||||
Module contains events processing mechanisms that are integrated with the standard python logging.
|
||||
|
||||
Example of usage:
|
||||
|
||||
::
|
||||
|
||||
from torch.distributed.elastic import events
|
||||
|
||||
event = events.Event(
|
||||
name="test_event", source=events.EventSource.WORKER, metadata={...}
|
||||
)
|
||||
events.get_logging_handler(destination="console").info(event)
|
||||
|
||||
"""
|
||||
|
||||
import inspect
|
||||
import logging
|
||||
import os
|
||||
import socket
|
||||
import traceback
|
||||
from typing import Optional
|
||||
|
||||
from torch.distributed.elastic.events.handlers import get_logging_handler
|
||||
|
||||
from .api import ( # noqa: F401
|
||||
Event,
|
||||
EventMetadataValue,
|
||||
EventSource,
|
||||
NodeState,
|
||||
RdzvEvent,
|
||||
)
|
||||
|
||||
|
||||
_events_loggers: dict[str, logging.Logger] = {}
|
||||
|
||||
|
||||
def _get_or_create_logger(destination: str = "null") -> logging.Logger:
|
||||
"""
|
||||
Construct python logger based on the destination type or extends if provided.
|
||||
|
||||
Available destination could be found in ``handlers.py`` file.
|
||||
The constructed logger does not propagate messages to the upper level loggers,
|
||||
e.g. root logger. This makes sure that a single event can be processed once.
|
||||
|
||||
Args:
|
||||
destination: The string representation of the event handler.
|
||||
Available handlers found in ``handlers`` module
|
||||
"""
|
||||
global _events_loggers
|
||||
|
||||
if destination not in _events_loggers:
|
||||
_events_logger = logging.getLogger(f"torchelastic-events-{destination}")
|
||||
_events_logger.setLevel(os.environ.get("LOGLEVEL", "INFO"))
|
||||
# Do not propagate message to the root logger
|
||||
_events_logger.propagate = False
|
||||
|
||||
logging_handler = get_logging_handler(destination)
|
||||
_events_logger.addHandler(logging_handler)
|
||||
|
||||
# Add the logger to the global dictionary
|
||||
_events_loggers[destination] = _events_logger
|
||||
|
||||
return _events_loggers[destination]
|
||||
|
||||
|
||||
def record(event: Event, destination: str = "null") -> None:
|
||||
_get_or_create_logger(destination).info(event.serialize())
|
||||
|
||||
|
||||
def record_rdzv_event(event: RdzvEvent) -> None:
|
||||
_get_or_create_logger("dynamic_rendezvous").info(event.serialize())
|
||||
|
||||
|
||||
def construct_and_record_rdzv_event(
|
||||
run_id: str,
|
||||
message: str,
|
||||
node_state: NodeState,
|
||||
name: str = "",
|
||||
hostname: str = "",
|
||||
pid: int | None = None,
|
||||
master_endpoint: str = "",
|
||||
local_id: int | None = None,
|
||||
rank: int | None = None,
|
||||
) -> None:
|
||||
"""
|
||||
Initialize rendezvous event object and record its operations.
|
||||
|
||||
Args:
|
||||
run_id (str): The run id of the rendezvous.
|
||||
message (str): The message describing the event.
|
||||
node_state (NodeState): The state of the node (INIT, RUNNING, SUCCEEDED, FAILED).
|
||||
name (str): Event name. (E.g. Current action being performed).
|
||||
hostname (str): Hostname of the node.
|
||||
pid (Optional[int]): The process id of the node.
|
||||
master_endpoint (str): The master endpoint for the rendezvous store, if known.
|
||||
local_id (Optional[int]): The local_id of the node, if defined in dynamic_rendezvous.py
|
||||
rank (Optional[int]): The rank of the node, if known.
|
||||
Returns:
|
||||
None
|
||||
Example:
|
||||
>>> # See DynamicRendezvousHandler class
|
||||
>>> def _record(
|
||||
... self,
|
||||
... message: str,
|
||||
... node_state: NodeState = NodeState.RUNNING,
|
||||
... rank: Optional[int] = None,
|
||||
... ) -> None:
|
||||
... construct_and_record_rdzv_event(
|
||||
... name=f"{self.__class__.__name__}.{get_method_name()}",
|
||||
... run_id=self._settings.run_id,
|
||||
... message=message,
|
||||
... node_state=node_state,
|
||||
... hostname=self._this_node.addr,
|
||||
... pid=self._this_node.pid,
|
||||
... local_id=self._this_node.local_id,
|
||||
... rank=rank,
|
||||
... )
|
||||
"""
|
||||
# We don't want to perform an extra computation if not needed.
|
||||
if isinstance(get_logging_handler("dynamic_rendezvous"), logging.NullHandler):
|
||||
return
|
||||
|
||||
# Set up parameters.
|
||||
if not hostname:
|
||||
hostname = socket.getfqdn()
|
||||
if not pid:
|
||||
pid = os.getpid()
|
||||
|
||||
# Determines which file called this function.
|
||||
callstack = inspect.stack()
|
||||
filename = "no_file"
|
||||
if len(callstack) > 1:
|
||||
stack_depth_1 = callstack[1]
|
||||
filename = os.path.basename(stack_depth_1.filename)
|
||||
if not name:
|
||||
name = stack_depth_1.function
|
||||
|
||||
# Delete the callstack variable. If kept, this can mess with python's
|
||||
# garbage collector as we are holding on to stack frame information in
|
||||
# the inspect module.
|
||||
del callstack
|
||||
|
||||
# Set up error trace if this is an exception
|
||||
if node_state == NodeState.FAILED:
|
||||
error_trace = traceback.format_exc()
|
||||
else:
|
||||
error_trace = ""
|
||||
|
||||
# Initialize event object
|
||||
event = RdzvEvent(
|
||||
name=f"{filename}:{name}",
|
||||
run_id=run_id,
|
||||
message=message,
|
||||
hostname=hostname,
|
||||
pid=pid,
|
||||
node_state=node_state,
|
||||
master_endpoint=master_endpoint,
|
||||
rank=rank,
|
||||
local_id=local_id,
|
||||
error_trace=error_trace,
|
||||
)
|
||||
|
||||
# Finally, record the event.
|
||||
record_rdzv_event(event)
|
||||
@@ -0,0 +1,116 @@
|
||||
#!/usr/bin/env python3
|
||||
# mypy: allow-untyped-defs
|
||||
|
||||
# Copyright (c) Facebook, Inc. and its affiliates.
|
||||
# All rights reserved.
|
||||
#
|
||||
# This source code is licensed under the BSD-style license found in the
|
||||
# LICENSE file in the root directory of this source tree.
|
||||
|
||||
import json
|
||||
from dataclasses import asdict, dataclass, field
|
||||
from enum import Enum
|
||||
from typing import Union
|
||||
|
||||
|
||||
__all__ = ["EventSource", "Event", "NodeState", "RdzvEvent"]
|
||||
|
||||
EventMetadataValue = str | int | float | bool | None
|
||||
|
||||
|
||||
class EventSource(str, Enum):
|
||||
"""Known identifiers of the event producers."""
|
||||
|
||||
AGENT = "AGENT"
|
||||
WORKER = "WORKER"
|
||||
|
||||
|
||||
@dataclass
|
||||
class Event:
|
||||
"""
|
||||
The class represents the generic event that occurs during the torchelastic job execution.
|
||||
|
||||
The event can be any kind of meaningful action.
|
||||
|
||||
Args:
|
||||
name: event name.
|
||||
source: the event producer, e.g. agent or worker
|
||||
timestamp: timestamp in milliseconds when event occurred.
|
||||
metadata: additional data that is associated with the event.
|
||||
"""
|
||||
|
||||
name: str
|
||||
source: EventSource
|
||||
timestamp: int = 0
|
||||
metadata: dict[str, EventMetadataValue] = field(default_factory=dict)
|
||||
|
||||
def __str__(self):
|
||||
return self.serialize()
|
||||
|
||||
@staticmethod
|
||||
def deserialize(data: Union[str, "Event"]) -> "Event":
|
||||
if isinstance(data, Event):
|
||||
return data
|
||||
if isinstance(data, str):
|
||||
data_dict = json.loads(data)
|
||||
data_dict["source"] = EventSource[data_dict["source"]] # type: ignore[possibly-undefined]
|
||||
# pyrefly: ignore [unbound-name]
|
||||
return Event(**data_dict)
|
||||
|
||||
def serialize(self) -> str:
|
||||
return json.dumps(asdict(self))
|
||||
|
||||
|
||||
class NodeState(str, Enum):
|
||||
"""The states that a node can be in rendezvous."""
|
||||
|
||||
INIT = "INIT"
|
||||
RUNNING = "RUNNING"
|
||||
SUCCEEDED = "SUCCEEDED"
|
||||
FAILED = "FAILED"
|
||||
|
||||
|
||||
@dataclass
|
||||
class RdzvEvent:
|
||||
"""
|
||||
Dataclass to represent any rendezvous event.
|
||||
|
||||
Args:
|
||||
name: Event name. (E.g. Current action being performed)
|
||||
run_id: The run id of the rendezvous
|
||||
message: The message describing the event
|
||||
hostname: Hostname of the node
|
||||
pid: The process id of the node
|
||||
node_state: The state of the node (INIT, RUNNING, SUCCEEDED, FAILED)
|
||||
master_endpoint: The master endpoint for the rendezvous store, if known
|
||||
rank: The rank of the node, if known
|
||||
local_id: The local_id of the node, if defined in dynamic_rendezvous.py
|
||||
error_trace: Error stack trace, if this is an error event.
|
||||
"""
|
||||
|
||||
name: str
|
||||
run_id: str
|
||||
message: str
|
||||
hostname: str
|
||||
pid: int
|
||||
node_state: NodeState
|
||||
master_endpoint: str = ""
|
||||
rank: int | None = None
|
||||
local_id: int | None = None
|
||||
error_trace: str = ""
|
||||
|
||||
def __str__(self):
|
||||
return self.serialize()
|
||||
|
||||
@staticmethod
|
||||
def deserialize(data: Union[str, "RdzvEvent"]) -> "RdzvEvent":
|
||||
if isinstance(data, RdzvEvent):
|
||||
return data
|
||||
if isinstance(data, str):
|
||||
data_dict = json.loads(data)
|
||||
data_dict["node_state"] = NodeState[data_dict["node_state"]] # type: ignore[possibly-undefined]
|
||||
# pyrefly: ignore [unbound-name]
|
||||
return RdzvEvent(**data_dict)
|
||||
|
||||
def serialize(self) -> str:
|
||||
return json.dumps(asdict(self))
|
||||
@@ -0,0 +1,21 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
# Copyright (c) Facebook, Inc. and its affiliates.
|
||||
# All rights reserved.
|
||||
#
|
||||
# This source code is licensed under the BSD-style license found in the
|
||||
# LICENSE file in the root directory of this source tree.
|
||||
|
||||
import logging
|
||||
|
||||
|
||||
_log_handlers: dict[str, logging.Handler] = {
|
||||
"console": logging.StreamHandler(),
|
||||
"dynamic_rendezvous": logging.NullHandler(),
|
||||
"null": logging.NullHandler(),
|
||||
}
|
||||
|
||||
|
||||
def get_logging_handler(destination: str = "null") -> logging.Handler:
|
||||
global _log_handlers
|
||||
return _log_handlers[destination]
|
||||
Reference in New Issue
Block a user